gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

item_uthash -> item_rcu && add foreign cont dir API

Browse Source
This commit is contained in:
liuwentan
2023-03-15 11:36:54 +08:00
parent 33c9c10467
commit 90d0764845
41 changed files with 2789 additions and 1603 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
deps/log/log.c vendored
View File

@@ -29,26 +29,24 @@
#include "log.h"
#define MAX_CALLBACKS 32
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
typedef struct {
log_func_t fn;
void *user_data;
int level;
} Callback;
typedef enum {
LOG_OP_IFACE_CONSOLE,
LOG_OP_IFACE_FILE,
RT_LOG_OP_IFACE_MAX,
}log_op_iface;
struct log_handle
{
int level;
int enable;
void *user_data;
log_lock_func_t lock;
char run_log_path[1024];
char cur_log_file[1024];
FILE *fp;
va_list ap;
char defined_log_fn[1024];
char runtime_log_fn[1024];
pthread_mutex_t mutex;
Callback callbacks[MAX_CALLBACKS];
log_op_iface iface;
};
static unsigned char weekday_str[7][4] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"};
@@ -78,29 +76,28 @@ static int log_create_dir(const char *dir_path, int path_len)
return ret;
}
static void log_close_file(struct log_event *event)
static void log_close_file(struct log_handle *handle)
{
pthread_mutex_lock(&event->mutex);
FILE *fp=(FILE *)event->user_data;
if(fp != NULL)
pthread_mutex_lock(&handle->mutex);
if(handle->fp != NULL)
{
fclose(fp);
event->user_data = NULL;
fclose(handle->fp);
handle->fp = NULL;
}
pthread_mutex_unlock(&event->mutex);
pthread_mutex_unlock(&handle->mutex);
return;
}
int log_open_file(char *file_name, struct log_event *event)
int log_open_file(char *file_name, struct log_handle *handle)
{
FILE *fp = NULL;
log_close_file(event);
log_close_file(handle);
if(NULL == (fp = fopen(file_name, "a")))
{
return -1;
}
memcpy(event->cur_log_file, file_name, strlen(file_name));
event->user_data = fp;
memcpy(handle->runtime_log_fn, file_name, strlen(file_name));
handle->fp = fp;
return 0;
}
@@ -137,61 +134,38 @@ static int log_create_path(const char *file_path)
return 0;
}
fclose(fp);
return 1;
}
int log_create_log_file(struct log_event *event)
int log_create_log_file(struct log_handle *handle)
{
time_t t;
struct tm local_time;
char tmp_log_file_name[512];
FILE *fp = (FILE *)event->user_data;
char tmp_log_file_name[1024+128];
time(&t);
if(NULL == (localtime_r(&t, &local_time)))
{
return 0;
}
sprintf(tmp_log_file_name, "%s.%04d-%02d-%02d", event->run_log_path, local_time.tm_year + 1900, local_time.tm_mon + 1, local_time.tm_mday);
snprintf(tmp_log_file_name, sizeof(tmp_log_file_name), "%s.%04d-%02d-%02d", handle->defined_log_fn, local_time.tm_year + 1900, local_time.tm_mon + 1, local_time.tm_mday);
if(fp == NULL)
if(handle->fp == NULL)
{
if(0 != log_open_file(tmp_log_file_name, event)) return 0;
if(0 != log_open_file(tmp_log_file_name, handle)) return 0;
}
else
{
if (0 != memcmp(tmp_log_file_name, event->cur_log_file, strlen(tmp_log_file_name)))
if (0 != memcmp(tmp_log_file_name, handle->runtime_log_fn, strlen(tmp_log_file_name)))
{
if(0 != log_open_file(tmp_log_file_name, event))return 0;
if(0 != log_open_file(tmp_log_file_name, handle))return 0;
}
}
return 1;
}
static void log_print_file(struct log_event *event)
{
char buf[64];
time_t t;
struct tm local_time;
const char *level_str_map[]= {"TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"};
time(&t);
if(NULL == (localtime_r(&t, &local_time))) return;
snprintf(buf, sizeof(buf), "%s %s %d %02d:%02d:%02d %d", weekday_str[local_time.tm_wday],
month_str[local_time.tm_mon], local_time.tm_mday, local_time.tm_hour, local_time.tm_min, local_time.tm_sec, local_time.tm_year+1900);
log_create_log_file(event);
fprintf((FILE *)event->user_data, "%s, %s, %s, ", buf, level_str_map[event->level], event->module);
vfprintf((FILE *)event->user_data, event->fmt, event->ap);
fprintf((FILE *)event->user_data, "\n");
fflush((FILE *)event->user_data);
}
static void log_print_console(struct log_event *event)
static void log_print_file(struct log_handle *handle, int level, const char *module, va_list ap, const char *fmt)
{
char buf[64]={0};
time_t t;
@@ -202,148 +176,83 @@ static void log_print_console(struct log_event *event)
if(NULL == (localtime_r(&t, &local_time))) return;
snprintf(buf, sizeof(buf), "%s %s %d %02d:%02d:%02d %d", weekday_str[local_time.tm_wday],
month_str[local_time.tm_mon], local_time.tm_mday, local_time.tm_hour, local_time.tm_min, local_time.tm_sec, local_time.tm_year+1900);
fprintf((FILE *)event->user_data, "%s, %s, %s, ", buf, level_str_map[event->level], event->module);
vfprintf((FILE *)event->user_data, event->fmt, event->ap);
fprintf((FILE *)event->user_data, "\n");
fflush((FILE *)event->user_data);
log_create_log_file(handle);
fprintf(handle->fp, "%s, %s, %s, ", buf, level_str_map[level], module);
vfprintf(handle->fp, fmt, ap);
fprintf(handle->fp, "\n");
fflush(handle->fp);
}
void log_options_set_lock_fn(struct log_handle * handle, log_lock_func_t fn, void *user_data)
static void log_print_console(struct log_handle *handle, int level, const char *module, va_list ap, const char *fmt)
{
struct log_handle *_handle_t = (struct log_handle *)handle;
if(_handle_t != NULL)
{
_handle_t->lock = fn;
_handle_t->user_data = user_data;
}
char buf[64]={0};
time_t t;
struct tm local_time;
const char *level_str_map[]= {"TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"};
time(&t);
if(NULL == (localtime_r(&t, &local_time))) return;
snprintf(buf, sizeof(buf), "%s %s %d %02d:%02d:%02d %d", weekday_str[local_time.tm_wday],
month_str[local_time.tm_mon], local_time.tm_mday, local_time.tm_hour, local_time.tm_min, local_time.tm_sec, local_time.tm_year+1900);
fprintf(handle->fp, "%s, %s, %s, ", buf, level_str_map[level], module);
vfprintf(handle->fp, fmt, ap);
fprintf(handle->fp, "\n");
fflush(handle->fp);
}
void log_options_set_level(struct log_handle * handle, int level)
{
struct log_handle *_handle_t = (struct log_handle *)handle;
if(_handle_t != NULL)
if(handle != NULL)
{
_handle_t->level = level;
handle->level = level;
}
}
void log_options_set_enable(struct log_handle * handle, int enable)
{
struct log_handle *_handle_t = (struct log_handle *)handle;
if(_handle_t != NULL)
if(handle != NULL)
{
_handle_t->enable = enable;
handle->enable = enable;
}
}
int log_add_method_callback(void *handle, log_func_t fn, void *udata, int level)
{
int i=0;
struct log_handle *_handle_t = (struct log_handle *)handle;
if(_handle_t == NULL)
{
return -1;
}
for (i = 0; i < MAX_CALLBACKS; i++)
{
if (!_handle_t->callbacks[i].fn)
{
_handle_t->callbacks[i] = (Callback) { fn, udata, level };
return 0;
}
}
return -1;
}
int log_options_add_file_fp(struct log_handle *handle, int level)
{
int ret=0;
FILE *fp=NULL;
struct log_handle *_handle_t = (struct log_handle *)handle;
if(!_handle_t)
{
return -1;
}
ret =log_create_path(_handle_t->run_log_path);
if(ret==0)
{
fp=stderr;
}
return log_add_method_callback(handle, log_print_file, fp, level);
}
int log_options_add_fp(struct log_handle * handle, FILE *fp, int level)
{
return log_add_method_callback(handle, log_print_file, fp, level);
}
static void log_lock(void *handle)
{
struct log_handle *_handle_t = (struct log_handle *)handle;
if(_handle_t != NULL && _handle_t->lock)
{
_handle_t->lock(1, _handle_t->user_data);
}
}
static void log_unlock(void *handle)
{
struct log_handle *_handle_t = (struct log_handle *)handle;
if(_handle_t != NULL && _handle_t->lock)
{
_handle_t->lock(0, _handle_t->user_data);
}
}
static void log_init_event(struct log_event *event, void *user_data)
{
event->user_data = user_data;
}
struct log_handle *log_handle_create(const char *file_path, int level)
{
struct log_handle *_handle_t = ALLOC(struct log_handle, 1);
if(!_handle_t)
struct log_handle *handle = ALLOC(struct log_handle, 1);
if(!handle)
{
return NULL;
}
_handle_t->enable=1;
_handle_t->level = level;
strncpy(_handle_t->run_log_path, file_path, 1024);
pthread_mutex_init(&_handle_t->mutex,NULL);
handle->enable=1;
handle->level = level;
strncpy(handle->defined_log_fn, file_path, 1024);
pthread_mutex_init(&handle->mutex,NULL);
log_options_add_file_fp(_handle_t, level);
if(handle->enable)
{
log_create_path(handle->defined_log_fn);
}
return _handle_t;
return handle;
}
void log_handle_destroy(struct log_handle * handle)
{
int i=0;
struct log_handle *_handle_t = (struct log_handle *)handle;
if(!_handle_t)
if(!handle)
{
return;
}
if (_handle_t->user_data != NULL)
if(handle->iface == LOG_OP_IFACE_FILE && handle->fp != NULL)
{
free(_handle_t->user_data);
_handle_t->user_data = NULL;
fclose(handle->fp);
handle->fp=NULL;
}
for(i=0; i<MAX_CALLBACKS; i++)
{
if(_handle_t->callbacks && _handle_t->callbacks->user_data)
{
fclose(_handle_t->callbacks->user_data);
_handle_t->callbacks->user_data = NULL;
}
}
pthread_mutex_destroy(&(_handle_t->mutex));
pthread_mutex_destroy(&(handle->mutex));
free(handle);
handle = NULL;
return;
@@ -351,41 +260,21 @@ void log_handle_destroy(struct log_handle * handle)
void log_print(struct log_handle *handle, int level, const char *module, const char *fmt, ...)
{
struct log_event event;
memset(&event, 0, sizeof(event));
va_list ap;
struct log_handle *_handle_t = (struct log_handle *)handle;
event.fmt=fmt;
event.level=level;
event.module=module;
event.mutex=_handle_t->mutex;
event.cur_log_file=_handle_t->cur_log_file;
event.run_log_path=_handle_t->run_log_path;
if(_handle_t->enable != 1 && level >= _handle_t->level)
if(handle->enable != 1 && level >= handle->level)
{
log_init_event(&event, stderr);
va_start(event.ap, fmt);
log_print_console(&event);
va_end(event.ap);
handle->fp = stdout;
handle->iface = LOG_OP_IFACE_CONSOLE;
va_start(handle->ap, fmt);
log_print_console(handle, level, module, ap, fmt);
va_end(handle->ap);
}
log_lock(handle);
int i=0;
for(i=0; i<MAX_CALLBACKS && _handle_t->callbacks[i].fn; i++)
if (handle->enable == 1 && level >= handle->level)
{
Callback *cb = &_handle_t->callbacks[i];
if (level >= cb->level)
{
log_init_event(&event, cb->user_data);
va_start(event.ap, fmt);
cb->fn(&event);
va_end(event.ap);
cb->user_data = event.user_data;
handle->iface = LOG_OP_IFACE_FILE;
va_start(ap, fmt);
log_print_file(handle, level, module, ap, fmt);
va_end(ap);
}
}
log_unlock(handle);
}

21
deps/log/log.h vendored
View File

@@ -17,27 +17,11 @@ extern "C"
#include <stdarg.h>
#include <stdbool.h>
#include <time.h>
#include <pthread.h>
#define LOG_VERSION "0.1.0"
struct log_handle;
struct log_event
{
int level;
va_list ap;
const char *fmt;
const char *module;
char *run_log_path;
char *cur_log_file;
void *user_data;
pthread_mutex_t mutex;
};
typedef void (*log_func_t)(struct log_event *ev);
typedef void (*log_lock_func_t)(int lock, void *udata);
enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL };
#define log_debug(handle, module, fmt, ...) log_print(handle, LOG_DEBUG, module, fmt, ##__VA_ARGS__)
@@ -48,11 +32,6 @@ enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL };
#define log_fatal(handle, module, fmt, ...) log_print(handle, LOG_FATAL, module, fmt, ##__VA_ARGS__)
void log_print(struct log_handle *, int level, const char *module, const char *fmt, ...);
int log_options_add_file_fp(struct log_handle *, int level);
int log_options_add_fp(struct log_handle *, FILE *fp, int level);
void log_options_set_lock_fn(struct log_handle *, log_lock_func_t fn, void *user_data);
void log_options_set_enable(struct log_handle *, int enable);
void log_options_set_level(struct log_handle *, int level);

18
include/maat.h
View File

@@ -48,6 +48,12 @@ enum maat_scan_status {
MAAT_SCAN_HIT //scan hit compile
};
enum maat_update_type {
MAAT_UPDATE_TYPE_INVALID = 0,
MAAT_UPDATE_TYPE_FULL,
MAAT_UPDATE_TYPE_INC
};
struct ip_addr {
int ip_type; //4: IPv4, 6: IPv6
union {
@@ -70,6 +76,7 @@ struct ipv6_tuple {
unsigned short dport; /* network order */
};
/* update_type: MAAT_UPDATE_TYPE_FULL or MAAT_UPDATE_TYPE_INC */
typedef void maat_start_callback_t(int update_type, void *u_param);
typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para);
typedef void maat_finish_callback_t(void *u_para);
@@ -89,8 +96,9 @@ int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_t
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len);
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name);
int maat_options_set_deferred_load_on(struct maat_options *opts);
int maat_options_set_foreign_cont_dir(struct maat_options *opts, const char *dir);
int maat_options_set_iris(struct maat_options *opts, const char *full_directory,
const char *increment_directory);
@@ -196,6 +204,12 @@ int maat_scan_stream(struct maat_stream **stream, const char *data, int data_len
void maat_scan_stream_close(struct maat_stream **stream);
/* maat state API */
struct maat_state *maat_state_new(struct maat *instance, int thread_id);
void maat_state_reset(struct maat_state *state);
void maat_state_free(struct maat_state **state);
int maat_state_set_scan_district(struct maat *instance, struct maat_state **state,
const char *district, size_t district_len);
@@ -210,8 +224,6 @@ int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state,
int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state,
struct maat_hit_object *objs, size_t n_obj);
void maat_state_free(struct maat_state **state);
/* return hit object compile_id */
int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj);

67
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -62,6 +62,7 @@ struct adapter_hs {
size_t n_expr;
size_t n_patterns;
struct adapter_hs_runtime *hs_rt;
struct hs_tag *tag_map;
};
struct adapter_hs_stream {
@@ -97,9 +98,13 @@ struct pattern_attribute {
};
struct hs_tag {
char *key;
size_t key_len;
size_t n_pat_attr;
struct pattern_attribute *pat_attr;
void *user_tag;
UT_hash_handle hh;
};
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread,
@@ -222,10 +227,40 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt
FREE(hs_cd);
}
struct hs_tag *hs_tag_new(long long expr_id, size_t n_pattern)
{
struct hs_tag *tag = ALLOC(struct hs_tag, 1);
tag->key = ALLOC(char, sizeof(long long));
memcpy(tag->key, (char *)&expr_id, sizeof(long long));
tag->key_len = sizeof(long long);
tag->pat_attr = ALLOC(struct pattern_attribute, n_pattern);
tag->n_pat_attr = n_pattern;
return tag;
}
void hs_tag_free(struct hs_tag *tag)
{
if (NULL == tag) {
return;
}
if (tag->key != NULL) {
FREE(tag->key);
}
if (tag->pat_attr != NULL) {
FREE(tag->pat_attr);
}
FREE(tag);
}
struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
enum hs_pattern_type pattern_type,
size_t n_worker_thread,
and_expr_t *exprs, size_t n_expr,
struct hs_expr *exprs, size_t n_expr,
struct log_handle *logger)
{
if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) ||
@@ -268,13 +303,13 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
compile_data = adpt_hs_compile_data_new(pattern_num);
uint32_t pattern_index = 0;
struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr);
struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1);
hs_instance->tag_map = NULL;
struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr);
/* populate adpt_hs_compile_data and bool_expr */
for (size_t i = 0; i < n_expr; i++) {
struct hs_tag *hs_tag = ALLOC(struct hs_tag, 1);
hs_tag->pat_attr = ALLOC(struct pattern_attribute, exprs[i].n_patterns);
hs_tag->n_pat_attr = exprs[i].n_patterns;
struct hs_tag *hs_tag = hs_tag_new(exprs[i].expr_id, exprs[i].n_patterns);
hs_tag->user_tag = exprs[i].user_tag;
for (size_t j = 0; j < exprs[i].n_patterns; j++) {
@@ -306,12 +341,12 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
bool_exprs[i].expr_id = exprs[i].expr_id;
bool_exprs[i].item_num = exprs[i].n_patterns;
bool_exprs[i].user_tag = hs_tag;
HASH_ADD_KEYPTR(hh, hs_instance->tag_map, hs_tag->key, hs_tag->key_len, hs_tag);
}
compile_data->n_patterns = pattern_index;
int ret = -1;
size_t mem_size = 0;
struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1);
hs_instance->n_worker_thread = n_worker_thread;
hs_instance->n_patterns = pattern_index;
@@ -325,6 +360,7 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
// printf("item[%zu] item_id: %llu\n", j, exprs[i].items[j].item_id);
// }
// }
/* create bool matcher */
hs_instance->hs_rt->bm = bool_matcher_new(bool_exprs, n_expr, &mem_size);
if (hs_instance->hs_rt->bm != NULL) {
@@ -334,7 +370,10 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
} else {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Adapter_hs module: build bool matcher failed",
__FUNCTION__, __LINE__);
goto error;
adpt_hs_compile_data_free(compile_data, pattern_index);
FREE(bool_exprs);
adapter_hs_destroy(hs_instance);
return NULL;
}
FREE(bool_exprs);
@@ -344,17 +383,15 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
goto error;
}
if (compile_data != NULL) {
adpt_hs_compile_data_free(compile_data, pattern_index);
}
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, pattern_type, logger);
if (ret < 0) {
goto error;
}
adpt_hs_compile_data_free(compile_data, pattern_index);
return hs_instance;
error:
adpt_hs_compile_data_free(compile_data, pattern_index);
adapter_hs_destroy(hs_instance);
return NULL;
@@ -391,6 +428,14 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance)
FREE(hs_instance->hs_rt);
}
if (hs_instance->tag_map != NULL) {
struct hs_tag *tag = NULL, *tmp_tag = NULL;
HASH_ITER(hh, hs_instance->tag_map, tag, tmp_tag) {
HASH_DEL(hs_instance->tag_map, tag);
hs_tag_free(tag);
}
}
FREE(hs_instance);
}

14
scanner/adapter_hs/adapter_hs.h
View File

@@ -58,7 +58,7 @@ struct hs_scan_result {
void *user_tag;
};
typedef struct {
struct hs_pattern {
enum hs_case_sensitive case_sensitive;
enum hs_match_mode match_mode;
int is_hexbin; /* 1(yes) 0(no) */
@@ -76,15 +76,15 @@ typedef struct {
char *pat;
/* pattern length */
size_t pat_len;
} scan_pattern_t;
};
/* logic AND expression, such as (pattern1 & pattern2) */
typedef struct {
uint32_t expr_id;
struct hs_expr {
long long expr_id;
size_t n_patterns;
scan_pattern_t patterns[MAX_EXPR_PATTERN_NUM];
struct hs_pattern patterns[MAX_EXPR_PATTERN_NUM];
void *user_tag;
} and_expr_t;
};
/**
* @brief initialize adapter_hs instance
@@ -99,7 +99,7 @@ typedef struct {
struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
enum hs_pattern_type pattern_type,
size_t n_worker_thread,
and_expr_t *exprs, size_t n_expr,
struct hs_expr *exprs, size_t n_expr,
struct log_handle *logger);
/**

12
src/inc_internal/maat_command.h
View File

@@ -24,8 +24,7 @@ enum maat_operation {
MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after
};
struct maat_cmd_line
{
struct maat_cmd_line {
const char *table_name;
const char *table_line;
long long rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary.
@@ -43,6 +42,15 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int increment);
long long maat_cmd_get_config_version(struct maat *maat_instance);
/* True(1), False(0) */
int maat_cmd_config_is_updating(struct maat *maat_instance);
char *maat_cmd_str_escape(char *dst, int size, const char *src);
int maat_cmd_flushDB(struct maat *maat_instance);
#ifdef __cplusplus
}
#endif

3
src/inc_internal/maat_common.h
View File

@@ -23,10 +23,9 @@ extern "C"
struct maat_options {
char instance_name[NAME_MAX];
char foreign_cont_dir[NAME_MAX];
size_t nr_worker_threads;
char *accept_tags;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
int gc_timeout_ms;

5
src/inc_internal/maat_compile.h
View File

@@ -17,6 +17,7 @@ extern "C"
#endif
#include "cJSON/cJSON.h"
#include "rcu_hash.h"
#include "maat.h"
#include "maat_kv.h"
#include "maat_rule.h"
@@ -49,6 +50,8 @@ void *compile_runtime_new(void *compile_schema, int max_thread_num,
struct log_handle *logger);
void compile_runtime_free(void *compile_runtime);
void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt);
int compile_runtime_update(void *compile_runtime, void *compile_schema,
const char *line, int valid_column);
int compile_runtime_commit(void *compile_runtime, const char *table_name);
@@ -82,7 +85,7 @@ struct maat_compile_state;
struct maat_compile_state *maat_compile_state_new(int thread_id);
void maat_compile_state_free(struct maat_compile_state *compile_state);
int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id,
long long *hit_item_ids, size_t hit_item_cnt,
size_t *n_hit_group_id, struct maat_state *state);

48
src/inc_internal/maat_ex_data.h
View File

@@ -18,18 +18,13 @@ extern "C"
#include "maat.h"
#include "rcu_hash.h"
#include "maat_garbage_collection.h"
struct ex_data_container {
struct ex_container {
void *ex_data;
void *custom_data;
};
struct ex_container_ctx {
int table_id;
void (*custom_data_free)(void *custom_data);
struct ex_data_schema *ex_schema;
};
struct ex_data_schema {
maat_ex_new_func_t *new_func;
maat_ex_free_func_t *free_func;
@@ -38,12 +33,18 @@ struct ex_data_schema {
void *argp;
};
struct ex_container_schema {
int table_id;
void (*custom_data_free)(void *custom_data);
struct ex_data_schema *ex_schema;
};
struct ex_data_runtime;
/* ex_data_runtime API */
struct ex_data_runtime *
ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn,
struct log_handle *logger);
struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt);
void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt);
@@ -67,40 +68,43 @@ void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt,
struct ex_data_schema *schema);
/* set user_ctx API */
void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt,
struct ex_container_ctx *container_ctx);
void ex_data_runtime_set_ex_container_schema(struct ex_data_runtime *ex_data_rt,
struct ex_container_schema *container_schema);
struct ex_container_ctx *
ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt);
struct ex_container_schema *
ex_data_runtime_get_ex_container_schema(struct ex_data_runtime *ex_data_rt);
struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data);
void ex_data_container_free(void *ctx, void *data);
struct ex_container *ex_container_new(void *ex_data, void *custom_data);
void ex_container_free(void *ctx, void *data);
/* ex_data_runtime ex data API */
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row,
const char *key, size_t key_len);
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt,
const char *row, const char *key,
size_t key_len);
int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len,
struct ex_data_container *ex_container);
struct ex_container *ex_container);
int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container ***ex_container);
size_t ex_data_runtime_list_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_container ***ex_container);
size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt);
int ex_data_runtime_is_updating(struct ex_data_runtime *ex_data_rt);
void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container *ex_container);
struct ex_container *ex_container);
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt);
#ifdef __cplusplus
}
#endif

2
src/inc_internal/maat_ip.h
View File

@@ -35,8 +35,6 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
const char *line, int valid_column);
int ip_runtime_commit(void *ip_runtime, const char *table_name);
struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt);
/* ip runtime scan API */
int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
uint8_t *ip_addr, int vtable_id, struct maat_state *state);

8
src/inc_internal/maat_kv.h
View File

@@ -24,11 +24,13 @@ struct maat_kv_store *maat_kv_store_new(void);
void maat_kv_store_free(struct maat_kv_store *store);
int maat_kv_register(struct maat_kv_store *store, const char *key, int value);
int maat_kv_register(struct maat_kv_store *store, const char *key, long long value);
int maat_kv_read(struct maat_kv_store *store, const char *key, int *value);
int maat_kv_read(struct maat_kv_store *store, const char *key, long long *value);
int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, int *value);
int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, long long *value);
int maat_kv_write(struct maat_kv_store *store, const char *key, long long value);
struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store);

27
src/inc_internal/maat_rule.h
View File

@@ -37,14 +37,14 @@ extern "C"
#define MAX_COMPILE_TABLE_NUM 16
#define MAX_PHYSICAL_TABLE_NUM 16
#define MAAT_UPDATE_TYPE_NONE 0
#define MAAT_UPDATE_TYPE_FULL 1
#define MAAT_UPDATE_TYPE_INC 2
#define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2
#define MAX_DISTRICT_STR 128
#define INVALID_VERSION -1
#define mr_region_id_var "SEQUENCE_REGION"
#define mr_group_id_var "SEQUENCE_GROUP"
enum last_scan_flag {
LAST_SCAN_UNSET,
@@ -63,7 +63,7 @@ struct maat_item_inner {
long long magic_num;
long long item_id;
long long group_id;
int district_id;
long long district_id;
int expr_id_cnt;
int expr_id_lb; //low boundary
int expr_id_ub; //up boundary
@@ -72,8 +72,9 @@ struct maat_item_inner {
struct maat_item {
long long item_id;
long long group_id;
UT_hash_handle hh;
void *user_data;
void (*user_data_free)(void *data);
};
#define COMPILE_RULE_MAGIC 0x1a2b3c4d
@@ -105,6 +106,7 @@ struct maat_runtime {
size_t max_table_num;
uint32_t rule_num;
struct maat_kv_store *sequence_map;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
@@ -216,6 +218,9 @@ struct maat {
long long load_specific_version; //Default: Load the Latest. Only valid in redis mode, and maybe failed for too old
char foreign_cont_dir[NAME_MAX];
/* internal state */
long long new_version;
/* statistics */
long long line_cmd_acc_num;
@@ -235,7 +240,7 @@ struct maat_state {
char compile_tables[MAX_COMPILE_TABLE_NUM][NAME_MAX]; //caller can select compile table to scan
unsigned char is_set_district;
unsigned char is_last_scan;
int district_id; //-1: Any District; -2: Unkonwn District;
long long district_id; //-1: Any District; -2: Unkonwn District;
int scan_cnt;
struct maat_compile_state *compile_state;
};
@@ -246,11 +251,11 @@ size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_
int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag);
struct maat_item *maat_item_new(long long item_id, long long group_id, void *user_data);
struct maat_item *maat_item_new(long long item_id, long long group_id, void *user_data, void (*user_data_free)(void *));
void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *));
void maat_item_free(void *user_ctx, void *maat_item);
struct maat_item_inner *maat_item_inner_new(long long group_id, long long item_id, int district_id);
struct maat_item_inner *maat_item_inner_new(long long group_id, long long item_id, long long district_id);
void maat_item_inner_free(void *item_inner);
@@ -262,6 +267,8 @@ void maat_finish_cb(void *u_para);
void *rule_monitor_loop(void *arg);
long long maat_runtime_get_sequence(struct maat_runtime *maat_rt, const char *key);
void maat_read_full_config(struct maat *maat_instance);
/* maat command API for internal */

4
src/inc_internal/maat_table.h
View File

@@ -68,8 +68,8 @@ size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr);
int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags);
int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str,
size_t district_str_len, int *district_id);
int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district);
size_t district_str_len, long long *district_id);
long long table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district);
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);

19
src/inc_internal/rcu_hash.h
View File

@@ -50,8 +50,23 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len
*/
void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len);
void *rcu_hash_updating_find(struct rcu_hash_table *htable, const char *key, size_t key_len);
/**
* @brief list all effective nodes
*
* @retval the number of effective nodes
*/
size_t rcu_hash_list(struct rcu_hash_table *htable, void ***data_array);
size_t rcu_hash_count(struct rcu_hash_table *htable);
size_t rcu_hash_updating_count(struct rcu_hash_table *htable);
/**
* @brief if rcu hash is updating
*
* @retval True(1) False(0)
*/
int rcu_hash_is_updating(struct rcu_hash_table *htable);
/**
* @brief make add/del effective
@@ -60,8 +75,6 @@ void rcu_hash_commit(struct rcu_hash_table *htable);
size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable);
size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array);
#ifdef __cplusplus
}
#endif

14
src/json2iris.c
View File

@@ -25,10 +25,6 @@
#define MODULE_JSON2IRIS module_name_str("maat.json2iris")
#define MAX_COLUMN_NUM 32
#define mr_region_id_var "SEQUENCE_REGION"
#define mr_group_id_var "SEQUENCE_GROUP"
#define MAX_PATH_LINE 512
const int json_version = 1;
@@ -104,7 +100,7 @@ struct iris_table *query_table_info(struct iris_description *p_iris,
strlen(table_name)));
snprintf(table_info->table_path, sizeof(table_info->table_path),
"%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name);
HASH_ADD_KEYPTR(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info);
HASH_ADD_KEYPTR(hh, p_iris->iris_table_map, table_info->table_name, strlen(table_name), table_info);
}
return table_info;
@@ -315,7 +311,7 @@ static struct group_info *group_info_add_unsafe(struct iris_description *p_iris,
group_info = ALLOC(struct group_info, 1);
group_info->group_id = get_group_seq(p_iris);
strncpy(group_info->group_name, group_name, sizeof(group_info->group_name));
HASH_ADD_KEYPTR(hh, p_iris->group_name_map, group_name, strlen(group_name), group_info);
HASH_ADD_KEYPTR(hh, p_iris->group_name_map, group_info->group_name, strlen(group_name), group_info);
}
return group_info;
}
@@ -364,7 +360,7 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int,
}
if (1 == cmd[i].str2int_flag) {
int int_value = 0;
long long int_value = 0;
char *p = item->valuestring;
ret = maat_kv_read(str2int, p, &int_value);
if (ret < 0) {
@@ -376,7 +372,7 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int,
goto error_out;
}
cmd[i].json_value = (char *)malloc(21);/* 2^64+1 can be represented in 21 chars. */
snprintf(cmd[i].json_value, 21, "%d", int_value);
snprintf(cmd[i].json_value, 21, "%d", (int)int_value);
} else {
switch (item->type) {
case cJSON_Number:
@@ -667,7 +663,7 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id,
const char *table_type_str = item->valuestring;
enum table_type table_type = TABLE_TYPE_EXPR;
int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type));
int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (long long *)&(table_type));
if (ret != 1) {
log_error(logger, MODULE_JSON2IRIS,
"[%s:%d] compile rule %d table name %s's table_type %s invalid",

119
src/maat_api.c
View File

@@ -128,58 +128,90 @@ void maat_options_free(struct maat_options *opts)
int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread)
{
opts->nr_worker_threads = n_thread;
if (NULL == opts) {
return -1;
}
opts->nr_worker_threads = n_thread;
return 0;
}
int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_tags)
{
opts->accept_tags = maat_strdup(accept_tags);
if (NULL == opts || NULL == accept_tags) {
return -1;
}
opts->accept_tags = maat_strdup(accept_tags);
return 0;
}
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms)
{
opts->rule_effect_interval_ms = interval_ms;
if (NULL == opts || interval_ms < 0) {
return -1;
}
opts->rule_effect_interval_ms = interval_ms;
return 0;
}
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms)
{
opts->rule_update_checking_interval_ms = interval_ms;
if (NULL == opts || interval_ms < 0) {
return -1;
}
opts->rule_update_checking_interval_ms = interval_ms;
return 0;
}
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms)
{
opts->gc_timeout_ms = interval_ms;
if (NULL == opts || interval_ms < 0) {
return -1;
}
opts->gc_timeout_ms = interval_ms;
return 0;
}
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name,
size_t name_len)
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name)
{
memcpy(opts->instance_name, instance_name, name_len);
if (NULL == opts || NULL == instance_name ||
strlen(instance_name) >= NAME_MAX) {
return -1;
}
memcpy(opts->instance_name, instance_name, strlen(instance_name));
return 0;
}
int maat_options_set_deferred_load_on(struct maat_options *opts)
{
opts->deferred_load_on = 1;
if (NULL == opts) {
return -1;
}
opts->deferred_load_on = 1;
return 0;
}
int maat_options_set_foreign_cont_dir(struct maat_options *opts, const char *dir)
{
if (NULL == opts || strlen(dir) >= NAME_MAX) {
return -1;
}
memcpy(opts->foreign_cont_dir, dir, strlen(dir));
return 0;
}
int maat_options_set_iris(struct maat_options *opts, const char *full_directory,
const char *increment_directory)
{
if (strlen(full_directory) >= NAME_MAX || strlen(increment_directory) >= NAME_MAX) {
if (NULL == opts || strlen(full_directory) >= NAME_MAX ||
strlen(increment_directory) >= NAME_MAX) {
return -1;
}
@@ -345,6 +377,7 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
maat_instance->rule_update_checking_interval_ms = opts->rule_update_checking_interval_ms;
maat_instance->gc_timeout_ms = opts->gc_timeout_ms;
maat_instance->deferred_load = opts->deferred_load_on;
memcpy(maat_instance->foreign_cont_dir, opts->foreign_cont_dir, strlen(opts->foreign_cont_dir));
garbage_gc_timeout_s = (maat_instance->rule_effect_interval_ms / 1000) +
(maat_instance->gc_timeout_ms / 1000);
maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s);
@@ -593,10 +626,10 @@ void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, int ta
}
ex_data_runtime_set_schema(ex_data_rt, ex_data_schema);
struct ex_container_ctx *ex_ctx = ALLOC(struct ex_container_ctx, 1);
ex_ctx->table_id = table_id;
ex_ctx->ex_schema = ex_data_schema;
ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_ctx);
struct ex_container_schema *container_schema = ALLOC(struct ex_container_schema, 1);
container_schema->table_id = table_id;
container_schema->ex_schema = ex_data_schema;
ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema);
size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt);
for (size_t i = 0; i < n_cached_row; i++) {
@@ -897,7 +930,7 @@ struct maat_state *grab_state(struct maat_state **state, struct maat *maat_insta
alignment_int64_array_add(maat_instance->outer_state_cnt, thread_id, 1);
}
if (NULL == mid->compile_state) {
if ((thread_id >= 0) && (NULL == mid->compile_state)) {
mid->compile_state = maat_compile_state_new(thread_id);
alignment_int64_array_add(maat_instance->compile_state_cnt, thread_id, 1);
}
@@ -1720,6 +1753,39 @@ void maat_scan_stream_close(struct maat_stream **maat_stream)
FREE(stream);
}
struct maat_state *maat_state_new(struct maat *maat_instance, int thread_id)
{
return NULL;
}
void maat_state_reset(struct maat_state *state)
{
}
void maat_state_free(struct maat_state **state)
{
struct maat_state *mid = NULL;
if (NULL == *state) {
return;
}
mid = *state;
if (mid->thread_id >= 0) {
alignment_int64_array_add(mid->maat_instance->outer_state_cnt, mid->thread_id, -1);
}
if (mid->compile_state != NULL) {
maat_compile_state_free(mid->compile_state);
mid->compile_state = NULL;
alignment_int64_array_add(mid->maat_instance->compile_state_cnt, mid->thread_id, -1);
}
mid->maat_instance = NULL;
free(mid);
*state = NULL;
}
int maat_state_set_scan_district(struct maat *maat_instance,
struct maat_state **state,
const char *district,
@@ -1834,29 +1900,6 @@ int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state,
return 0;
}
void maat_state_free(struct maat_state **state)
{
struct maat_state *mid = NULL;
if (NULL == *state) {
return;
}
mid = *state;
if (mid->thread_id >= 0) {
alignment_int64_array_add(mid->maat_instance->outer_state_cnt, mid->thread_id, -1);
}
if (mid->compile_state != NULL) {
maat_compile_state_free(mid->compile_state);
mid->compile_state = NULL;
alignment_int64_array_add(mid->maat_instance->compile_state_cnt, mid->thread_id, -1);
}
mid->maat_instance = NULL;
free(mid);
*state = NULL;
}
int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj)
{
return 0;

70
src/maat_bool_plugin.c
View File

@@ -104,7 +104,7 @@ void bool_plugin_schema_free(void *bool_plugin_schema)
schema->ex_schema = NULL;
}
FREE(schema);
free(schema);
}
/* ip plugin table ex data API */
@@ -182,8 +182,9 @@ void *bool_plugin_runtime_new(void *bool_plugin_schema, int max_thread_num,
struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema;
struct bool_plugin_runtime *bool_plugin_rt = ALLOC(struct bool_plugin_runtime, 1);
bool_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free,
logger);
bool_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id,
ex_container_free,
garbage_bin, logger);
bool_plugin_rt->ref_garbage_bin = garbage_bin;
bool_plugin_rt->logger = logger;
@@ -226,7 +227,7 @@ int bool_plugin_runtime_update_row(struct bool_plugin_runtime *bool_plugin_rt,
} else {
// add
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)expr);
struct ex_container *ex_container = ex_container_new(ex_data, (void *)expr);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
return -1;
@@ -385,12 +386,6 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche
bool_plugin_expr_free(bool_expr);
}
return -1;
} else {
if (0 == is_valid) {
bool_plugin_rt->rule_num--;
} else {
bool_plugin_rt->rule_num++;
}
}
} else {
//ex_schema not set
@@ -407,49 +402,64 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
return -1;
}
int ret = 0;
struct ex_data_container **ex_container = NULL;
struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime;
struct ex_data_runtime *ex_data_rt = bool_plugin_rt->ex_data_rt;
if (NULL == ex_data_rt) {
return -1;
}
size_t expr_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (0 == expr_cnt) {
FREE(ex_container);
int updating_flag = ex_data_runtime_is_updating(ex_data_rt);
if (0 == updating_flag) {
return 0;
}
struct bool_expr *exprs = ALLOC(struct bool_expr, expr_cnt);
ex_data_runtime_commit(ex_data_rt);
for (size_t i = 0; i < expr_cnt; i++) {
exprs[i] = *(struct bool_expr *)(struct bool_expr *)ex_container[i]->custom_data;
assert(exprs[i].user_tag == ex_container[i] || NULL == exprs[i].user_tag);
exprs[i].user_tag = ex_container[i];
struct bool_expr *rules = NULL;
struct ex_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
if (rule_cnt > 0) {
rules = ALLOC(struct bool_expr, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct bool_expr *)(struct bool_expr *)ex_container[i]->custom_data;
assert(rules[i].user_tag == ex_container[i] || NULL == rules[i].user_tag);
rules[i].user_tag = ex_container[i];
}
}
struct bool_matcher *new_bool_matcher = NULL;
struct bool_matcher *old_bool_matcher = NULL;
size_t mem_used = 0;
log_info(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"table[%s] committing %zu bool_plugin rules for rebuilding bool_matcher engine",
table_name, expr_cnt);
table_name, rule_cnt);
new_bool_matcher = bool_matcher_new(exprs, expr_cnt, &mem_used);
int ret = 0;
size_t mem_used = 0;
struct bool_matcher *new_bool_matcher = NULL;
struct bool_matcher *old_bool_matcher = NULL;
new_bool_matcher = bool_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_bool_matcher) {
log_error(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table[%s] rebuild bool_matcher engine failed when update %zu bool_plugin rules",
__FUNCTION__, __LINE__, table_name, expr_cnt);
__FUNCTION__, __LINE__, table_name, rule_cnt);
ret = -1;
}
old_bool_matcher = bool_plugin_rt->matcher;
bool_plugin_rt->matcher = new_bool_matcher;
if (old_bool_matcher != NULL) {
maat_garbage_bagging(bool_plugin_rt->ref_garbage_bin, old_bool_matcher,
(void (*)(void*))bool_matcher_free);
ex_data_runtime_commit(ex_data_rt);
}
FREE(exprs);
bool_plugin_rt->rule_num = rule_cnt;
if (rules != NULL) {
FREE(rules);
}
if (ex_container != NULL) {
FREE(ex_container);
}
return ret;
}
@@ -485,7 +495,7 @@ int bool_plugin_runtime_get_ex_data(void *bool_plugin_runtime, unsigned long lon
int n_result = bool_matcher_match(bool_plugin_rt->matcher, item_ids, n_item, results, n_ex_data);
for (int i = 0; i < n_result; i++) {
ex_data_array[i] = ex_data_runtime_get_ex_data_by_container(bool_plugin_rt->ex_data_rt,
(struct ex_data_container *)results[i].user_tag);
(struct ex_container *)results[i].user_tag);
}
return n_result;

212
src/maat_command.c
View File

@@ -84,19 +84,118 @@ redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port,
return c;
}
struct s_rule_array
long long maat_cmd_read_redis_integer(const redisReply *reply)
{
int cnt;
int size;
struct serial_rule *array;
};
switch (reply->type) {
case REDIS_REPLY_INTEGER:
return reply->integer;
break;
case REDIS_REPLY_ARRAY:
assert(reply->element[0]->type == REDIS_REPLY_INTEGER);
return reply->element[0]->integer;
break;
case REDIS_REPLY_STRING:
return atoll(reply->str);
break;
default:
return -1;
break;
}
return 0;
}
void save_serial_rule(void *data, void *user)
int redis_flushDB(redisContext *ctx, int db_index, struct log_handle *logger)
{
struct s_rule_array *array = (struct s_rule_array *)user;
int i = array->cnt;
memcpy(&(array->array[i]), data, sizeof(struct serial_rule));
array->array[i].op = MAAT_OP_ADD;
long long maat_redis_version = 0;
redisReply *data_reply = maat_cmd_wrap_redis_command(ctx, "WATCH MAAT_VERSION");
freeReplyObject(data_reply);
data_reply = NULL;
data_reply = maat_cmd_wrap_redis_command(ctx, "GET MAAT_VERSION");
if (data_reply->type == REDIS_REPLY_NIL) {
maat_redis_version = 0;
} else {
maat_redis_version = maat_cmd_read_redis_integer(data_reply);
maat_redis_version++;
freeReplyObject(data_reply);
data_reply = NULL;
}
data_reply = maat_cmd_wrap_redis_command(ctx, "DBSIZE");
long long dbsize = maat_cmd_read_redis_integer(data_reply);
freeReplyObject(data_reply);
data_reply = NULL;
data_reply = maat_cmd_wrap_redis_command(ctx, "MULTI");
freeReplyObject(data_reply);
data_reply = NULL;
int append_cmd_cnt = 0;
redisAppendCommand(ctx, "FLUSHDB");
append_cmd_cnt++;
redisAppendCommand(ctx, "SET MAAT_VERSION %lld", maat_redis_version);
append_cmd_cnt++;
redisAppendCommand(ctx, "SET MAAT_PRE_VER %lld", maat_redis_version);
append_cmd_cnt++;
redisAppendCommand(ctx, "SET %s 1", mr_region_id_var);
append_cmd_cnt++;
redisAppendCommand(ctx, "SET %s 1", mr_group_id_var);
append_cmd_cnt++;
redisAppendCommand(ctx, "EXEC");
append_cmd_cnt++;
int ret = 0;
int redis_transaction_success = 1;
for (int i = 0; i < append_cmd_cnt; i++) {
ret = maat_cmd_wrap_redis_get_reply(ctx, &data_reply);
if (ret == REDIS_OK) {
if (0 == mr_transaction_success(data_reply)) {
redis_transaction_success = 0;
}
freeReplyObject(data_reply);
data_reply = NULL;
}
}
if (redis_transaction_success == 1) {
log_info(logger, MODULE_MAAT_COMMAND,
"FlushDB %d, MAAT_VERSION:%llu, DBSize:%llu.",
db_index, (maat_redis_version == 0)?0:(maat_redis_version-1),dbsize);
}
return redis_transaction_success;
}
redisContext *get_redis_ctx_for_write(struct maat *maat_instance)
{
if (NULL == maat_instance->mr_ctx.write_ctx) {
int ret = connect_redis_for_write(&(maat_instance->mr_ctx),
maat_instance->logger);
if(ret!=0)
{
return NULL;
}
}
return maat_instance->mr_ctx.write_ctx;
}
int maat_cmd_flushDB(struct maat *maat_instance)
{
int ret = 0;
redisContext *write_ctx = get_redis_ctx_for_write(maat_instance);
if (NULL == write_ctx) {
return -1;
}
do {
ret = redis_flushDB(maat_instance->mr_ctx.write_ctx,
maat_instance->mr_ctx.redis_db,
maat_instance->logger);
} while(0 == ret);
return 0;
}
void maat_cmd_clear_rule_cache(struct serial_rule *s_rule)
@@ -130,19 +229,6 @@ int connect_redis_for_write(struct source_redis_ctx *mr_ctx,
}
}
redisContext *get_redis_ctx_for_write(struct maat *maat_instance)
{
if (NULL == maat_instance->mr_ctx.write_ctx) {
int ret = connect_redis_for_write(&(maat_instance->mr_ctx),
maat_instance->logger);
if(ret!=0)
{
return NULL;
}
}
return maat_instance->mr_ctx.write_ctx;
}
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
long long rule_id, const char *table_name,
const char *line, long long timeout)
@@ -266,26 +352,6 @@ const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len)
return line + start;
}
long long maat_cmd_read_redis_integer(const redisReply *reply)
{
switch (reply->type) {
case REDIS_REPLY_INTEGER:
return reply->integer;
break;
case REDIS_REPLY_ARRAY:
assert(reply->element[0]->type == REDIS_REPLY_INTEGER);
return reply->element[0]->integer;
break;
case REDIS_REPLY_STRING:
return atoll(reply->str);
break;
default:
return -1;
break;
}
return 0;
}
int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply)
{
return redisGetReply(c, (void **)reply);
@@ -374,3 +440,63 @@ long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int incre
return result;
}
long long maat_cmd_get_config_version(struct maat *maat_instance)
{
long long new_version = -1;
if (maat_instance->new_version != INVALID_VERSION) {
new_version = maat_instance->new_version;
} else {
new_version = maat_instance->maat_version;
}
return new_version;
}
int maat_cmd_config_is_updating(struct maat *maat_instance)
{
int ret = 0;
if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
ret = 0;
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
} else {
ret = 1;
}
return ret;
}
char *maat_cmd_str_escape(char *dst, int size, const char *src)
{
int i = 0, j = 0;
int len = strlen(src);
for (i = 0, j = 0; i < len && j < size; i++) {
switch (src[i]) {
case '&':
dst[j] = '\\';
dst[j+1] = '&';
j += 2;
break;
case ' ':
dst[j] = '\\';
dst[j+1] = 'b';//space,0x20;
j += 2;
break;
case '\\':
dst[j] = '\\';
dst[j+1] = '\\';
j += 2;
break;
default:
dst[j] = src[i];
j++; //undo the followed i++
break;
}
}
dst[j] = '\0';
return dst;
}

213
src/maat_compile.c
View File

@@ -22,7 +22,6 @@
#include "maat_garbage_collection.h"
#include "maat_group.h"
#include "maat_ex_data.h"
#include "rcu_hash.h"
#include "maat_table.h"
#define MODULE_COMPILE module_name_str("maat.compile")
@@ -71,11 +70,11 @@ struct group2compile_item {
struct compile_runtime {
struct bool_matcher *bm;
struct maat_compile *compile_hash; // <compile_id, struct maat_compile>
unsigned long long clause_id_generator;
struct maat_runtime *ref_maat_rt;
uint32_t rule_num;
pthread_rwlock_t rwlock; /* TODO: replaced with mutex? */
struct bool_expr_match *expr_match_buff;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
@@ -88,10 +87,10 @@ struct group2compile_runtime {
};
struct maat_clause_state {
unsigned long long clause_id;
long long clause_id;
char not_flag;
char in_use;
UT_array *literal_ids;
UT_array *ut_literal_ids;
};
struct maat_literal_id {
@@ -100,7 +99,7 @@ struct maat_literal_id {
};
struct maat_clause {
unsigned long long clause_id;
long long clause_id;
size_t n_literal_id;
struct maat_literal_id *literal_ids;
UT_hash_handle hh;
@@ -510,8 +509,15 @@ void maat_compile_free(struct maat_compile *compile)
for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
clause_state = compile->clause_states + i;
utarray_free(clause_state->literal_ids);
clause_state->literal_ids = NULL;
// size_t literal_len = utarray_len(clause_state->ut_literal_ids);
// printf("maat_compile_free compile_id:%lld index:%d literal_len:%zu\n",
// compile->compile_id, i, literal_len);
if (clause_state->ut_literal_ids != NULL) {
utarray_free(clause_state->ut_literal_ids);
clause_state->ut_literal_ids = NULL;
}
clause_state->in_use = 0;
}
compile->magic = 0;
@@ -546,15 +552,29 @@ void compile_runtime_free(void *compile_runtime)
if (compile_rt->compile_hash != NULL) {
maat_compile_hash_free(&(compile_rt->compile_hash));
}
pthread_rwlock_unlock(&compile_rt->rwlock);
if (compile_rt->expr_match_buff != NULL) {
FREE(compile_rt->expr_match_buff);
}
pthread_rwlock_unlock(&compile_rt->rwlock);
pthread_rwlock_destroy(&compile_rt->rwlock);
FREE(compile_rt);
}
void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt)
{
if (NULL == compile_runtime) {
return;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
pthread_rwlock_wrlock(&compile_rt->rwlock);
compile_rt->ref_maat_rt = maat_rt;
pthread_rwlock_unlock(&compile_rt->rwlock);
}
void *group2compile_runtime_new(void *g2c_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
@@ -695,7 +715,7 @@ struct maat_compile *maat_compile_new(long long compile_id)
compile->compile_id = compile_id;
for(int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
utarray_new(compile->clause_states[i].literal_ids, &ut_literal_id_icd);
utarray_new(compile->clause_states[i].ut_literal_ids, &ut_literal_id_icd);
compile->clause_states[i].in_use=0;
}
@@ -771,7 +791,7 @@ int maat_compile_hash_remove(struct maat_compile **compile_hash, struct maat_com
size_t compile_cnt = HASH_COUNT(*compile_hash);
struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL;
HASH_ITER (hh, *compile_hash, compile1, tmp_compile1) {
printf("<maat_compile_hash_remove> compile_id:%lu, compile_cnt:%zu\n",
printf("<maat_compile_hash_remove> compile_id:%lld, compile_cnt:%zu\n",
compile1->compile_id, compile_cnt);
}
#endif
@@ -787,26 +807,6 @@ struct maat_compile *maat_compile_hash_find(struct maat_compile **compile_hash,
return compile;
}
size_t maat_compile_in_use_count(struct maat_compile *compile_hash)
{
struct maat_compile *compile = NULL, *tmp_compile = NULL;
size_t in_use_compile_cnt = 0;
struct maat_clause_state *clause_state = NULL;
HASH_ITER(hh, compile_hash, compile, tmp_compile) {
//find how much compile whose clause is in_use
for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
clause_state = compile->clause_states + i;
if (clause_state->in_use) {
in_use_compile_cnt++;
break;
}
}
}
return in_use_compile_cnt;
}
int compare_literal_id(const void *pa, const void *pb)
{
struct maat_literal_id *la = (struct maat_literal_id *)pa;
@@ -833,15 +833,15 @@ int maat_compile_clause_add_literal(struct maat_compile *compile,
}
struct maat_literal_id *tmp = NULL;
tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids,
tmp = (struct maat_literal_id *)utarray_find(clause_state->ut_literal_ids,
literal_id, compare_literal_id);
if (tmp) {
assert(tmp->group_id == literal_id->group_id);
assert(tmp->vtable_id == literal_id->vtable_id);
return -1;
} else {
utarray_push_back(clause_state->literal_ids, literal_id);
utarray_sort(clause_state->literal_ids, compare_literal_id);
utarray_push_back(clause_state->ut_literal_ids, literal_id);
utarray_sort(clause_state->ut_literal_ids, compare_literal_id);
}
return 0;
@@ -853,7 +853,7 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile,
{
struct maat_clause_state* clause_state = compile->clause_states + clause_index;
struct maat_literal_id *tmp = NULL;
tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids,
tmp = (struct maat_literal_id *)utarray_find(clause_state->ut_literal_ids,
literal_id, compare_literal_id);
if (tmp) {
assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id));
@@ -861,11 +861,11 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile,
return -1;
}
size_t remove_idx = utarray_eltidx(clause_state->literal_ids, tmp);
utarray_erase(clause_state->literal_ids, remove_idx, 1);
size_t remove_idx = utarray_eltidx(clause_state->ut_literal_ids, tmp);
utarray_erase(clause_state->ut_literal_ids, remove_idx, 1);
if (0 == utarray_len(clause_state->literal_ids)) {
clause_state->in_use=0;
if (0 == utarray_len(clause_state->ut_literal_ids)) {
clause_state->in_use = 0;
compile->actual_clause_num--;
}
@@ -874,7 +874,7 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile,
static const struct maat_clause *
maat_clause_hash_fetch_clause(struct maat_clause **clause_hash,
unsigned long long *clause_id_generator,
struct maat_runtime *ref_maat_rt,
struct maat_literal_id *literal_ids,
size_t n_literal_id)
{
@@ -883,12 +883,11 @@ maat_clause_hash_fetch_clause(struct maat_clause **clause_hash,
HASH_FIND(hh, *clause_hash, literal_ids, n_literal_id * sizeof(struct maat_literal_id), clause);
if (!clause) {
clause = ALLOC(struct maat_clause, 1);
clause->clause_id = *clause_id_generator;
clause->clause_id = maat_runtime_get_sequence(ref_maat_rt, "clause_id");
clause->n_literal_id = n_literal_id;
clause->literal_ids = ALLOC(struct maat_literal_id, n_literal_id);
memcpy(clause->literal_ids, literal_ids, n_literal_id * sizeof(struct maat_literal_id));
(*clause_id_generator)++;
HASH_ADD_KEYPTR(hh, *clause_hash, clause->literal_ids,
n_literal_id * sizeof(struct maat_literal_id), clause);
}
@@ -896,24 +895,21 @@ maat_clause_hash_fetch_clause(struct maat_clause **clause_hash,
return clause;
}
static void maat_clause_hash_free(struct maat_clause *clause_hash)
static void maat_clause_hash_free(struct maat_clause **clause_hash)
{
struct maat_clause *clause = NULL, *tmp_clause = NULL;
HASH_ITER (hh, clause_hash, clause, tmp_clause) {
HASH_DELETE(hh, clause_hash, clause);
free(clause->literal_ids);
HASH_ITER (hh, *clause_hash, clause, tmp_clause) {
HASH_DEL(*clause_hash, clause);
FREE(clause->literal_ids);
clause->n_literal_id = 0;
free(clause);
FREE(clause);
}
}
struct bool_matcher *
maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
unsigned long long *clause_id_generator,
struct log_handle *logger)
struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compile_rt)
{
if (NULL == compile_hash || NULL == logger) {
if (NULL == compile_rt) {
return NULL;
}
@@ -924,11 +920,12 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
const struct maat_clause *clause = NULL;
struct maat_clause *clause_hash = NULL; // <literal_id, struct maat_clause>
pthread_rwlock_rdlock(&compile_rt->rwlock);
//STEP 1, update clause_id of each compile and literal
struct maat_compile *compile = NULL, *tmp_compile = NULL;
struct maat_literal_id *literal_ids = NULL;
size_t n_literal_id = 0;
HASH_ITER(hh, compile_hash, compile, tmp_compile) {
HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) {
has_clause_num = 0;
for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
clause_state = compile->clause_states + i;
@@ -938,9 +935,9 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
}
has_clause_num++;
literal_ids = (struct maat_literal_id *)utarray_eltptr(clause_state->literal_ids, 0);
n_literal_id = utarray_len(clause_state->literal_ids);
clause = maat_clause_hash_fetch_clause(&clause_hash, clause_id_generator,
literal_ids = (struct maat_literal_id *)utarray_eltptr(clause_state->ut_literal_ids, 0);
n_literal_id = utarray_len(clause_state->ut_literal_ids);
clause = maat_clause_hash_fetch_clause(&clause_hash, compile_rt->ref_maat_rt,
literal_ids, n_literal_id);
clause_state->clause_id = clause->clause_id;
}
@@ -949,9 +946,9 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
//STEP 2, serial compile clause states to a bool expression array
size_t expr_cnt = 0;
size_t compile_cnt = maat_compile_in_use_count(compile_hash);
size_t compile_cnt = HASH_COUNT(compile_rt->compile_hash);
struct bool_expr *bool_expr_array = ALLOC(struct bool_expr, compile_cnt);
HASH_ITER(hh, compile_hash, compile, tmp_compile) {
HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) {
for (i = 0, j = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
if (compile->clause_states[i].in_use) {
if (compile->clause_states[i].not_flag) {
@@ -961,9 +958,9 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
//TODO:mytest need to delete
#if 0
struct maat_literal_id *p = NULL;
for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) {
printf("<before bool_matcher_new> compile_id:%lld, clause_id:%llu, literal{%lld: %d}\n",
compile->compile_id, compile->clause_states[i].clause_id, p->group_id, p->vtable_id);
for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].ut_literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].ut_literal_ids,p)) {
printf("<before bool_matcher_new> compile_rt:%p compile_id:%lld, clause_id:%llu, literal{%lld: %d}\n",
compile_rt, compile->compile_id, compile->clause_states[i].clause_id, p->group_id, p->vtable_id);
}
#endif
bool_expr_array[expr_cnt].items[j].item_id = compile->clause_states[i].clause_id;
@@ -973,7 +970,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
}
}
// printf("bool_matcher_new compile_id:%lu j:%zu, compile->declared_clause_num:%d\n",
// printf("bool_matcher_new compile_id:%lld j:%zu, compile->declared_clause_num:%d\n",
// compile->compile_id, j, compile->declared_clause_num);
//some compile may have zero groups, e.g. default policy.
if (j == (size_t)compile->declared_clause_num && j > 0) {
@@ -983,13 +980,13 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
expr_cnt++;
}
}
pthread_rwlock_unlock(&compile_rt->rwlock);
//size_t expr_index = 0, item_index = 0;
// STEP 3, build bool matcher
size_t mem_size = 0;
if (0 == expr_cnt) {
log_error(logger, MODULE_COMPILE,
log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] No bool expression to build bool matcher.",
__FUNCTION__, __LINE__);
goto error;
@@ -999,28 +996,32 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
#if 0
printf("bool_matcher_new....................expr_cnt:%zu\n", expr_cnt);
for (expr_index = 0; expr_index < expr_cnt; expr_index++) {
printf("bool_expr_array[%zu].expr_id:%llu, item_num:%zu\n", expr_index, bool_expr_array[expr_index].expr_id,
bool_expr_array[expr_index].item_num);
for (item_index = 0; item_index < bool_expr_array[expr_index].item_num; item_index++) {
printf("bool_expr_array[%zu].items[%zu]:%llu, not_flag:%d\n", expr_index, item_index,
bool_expr_array[expr_index].items[item_index].item_id,
bool_expr_array[expr_index].items[item_index].not_flag);
if (bool_expr_array[expr_index].expr_id == 141 || bool_expr_array[expr_index].expr_id == 197) {
printf("compile_rt:%p expr_id:%llu\n", compile_rt, bool_expr_array[expr_index].expr_id);
// for (item_index = 0; item_index < bool_expr_array[expr_index].item_num; item_index++) {
// printf("bool_expr_array[%zu].items[%zu]:%llu, not_flag:%d\n", expr_index, item_index,
// bool_expr_array[expr_index].items[item_index].item_id,
// bool_expr_array[expr_index].items[item_index].not_flag);
// }
}
printf("\n");
}
#endif
bm = bool_matcher_new(bool_expr_array, expr_cnt, &mem_size);
if (bm != NULL) {
log_info(logger, MODULE_COMPILE,
log_info(compile_rt->logger, MODULE_COMPILE,
"Build bool matcher of %zu expressions with %zu bytes memory.", expr_cnt, mem_size);
} else {
log_error(logger, MODULE_COMPILE, "[%s:%d] Build bool matcher failed!",
log_error(compile_rt->logger, MODULE_COMPILE, "[%s:%d] Build bool matcher failed!",
__FUNCTION__, __LINE__);
}
error:
maat_clause_hash_free(clause_hash);
if (clause_hash != NULL) {
maat_clause_hash_free(&clause_hash);
clause_hash = NULL;
}
FREE(bool_expr_array);
return bm;
@@ -1031,8 +1032,7 @@ void maat_compile_bool_matcher_free(struct bool_matcher *bm)
bool_matcher_free(bm);
}
static int maat_compile_has_clause(struct maat_compile *compile,
unsigned long long clause_id)
static int maat_compile_has_clause(struct maat_compile *compile, long long clause_id)
{
struct maat_clause_state *clause_state = NULL;
@@ -1056,9 +1056,9 @@ static size_t compile_state_if_new_hit_compile(struct maat_compile_state *compil
size_t r_in_c_cnt = 0;
int ret = 0;
unsigned long long new_hit_clause_id = 0;
long long new_hit_clause_id = 0;
for (size_t i = 0; i < utarray_len(compile_state->this_scan_hit_clauses); i++) {
new_hit_clause_id = *(unsigned long long*)utarray_eltptr(compile_state->this_scan_hit_clauses, i);
new_hit_clause_id = *(long long*)utarray_eltptr(compile_state->this_scan_hit_clauses, i);
ret = maat_compile_has_clause(compile, new_hit_clause_id);
if (ret) {
r_in_c_cnt++;
@@ -1068,25 +1068,25 @@ static size_t compile_state_if_new_hit_compile(struct maat_compile_state *compil
return r_in_c_cnt;
}
size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan,
size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int is_last_scan,
struct maat_compile_state *compile_state,
void **user_data_array, size_t ud_array_size)
{
struct maat_compile *compile = NULL;
struct bool_expr_match *expr_match = ALLOC(struct bool_expr_match, MAX_SCANNER_HIT_COMPILE_NUM);
size_t ud_result_cnt = 0;
struct maat_compile *compile = NULL;
struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM;
assert(compile_state->thread_id >= 0);
//TODO:mytest need to delete
#if 0
unsigned long long *p;
printf("utarray_len:%u\n", utarray_len(compile_state->all_hit_clauses));
unsigned long long *p = utarray_eltptr(compile_state->all_hit_clauses, 0);
for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clauses); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clauses, p))
{
printf("before bool_matcher_match compile_state clause_id:%llu\n", *p);
printf("before bool_matcher_match compile_rt:%p compile_state clause_id:%llu\n", compile_rt, *p);
}
#endif
int bool_match_ret = bool_matcher_match(bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
int bool_match_ret = bool_matcher_match(compile_rt->bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses),
expr_match, MAX_SCANNER_HIT_COMPILE_NUM);
for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) {
@@ -1112,6 +1112,7 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan
}
compile_state->this_scan_hit_item_cnt = 0;
return ud_result_cnt;
}
@@ -1143,7 +1144,7 @@ int maat_add_group_to_compile(struct maat_compile **compile_hash, struct group2c
ret = 0;
}
// printf("group2compile update compile_id:%lu, compile->declared_clause_num:%d\n",
// printf("group2compile update compile_id:%lld, compile->declared_clause_num:%d\n",
// compile->compile_id, compile->declared_clause_num);
return ret;
}
@@ -1167,7 +1168,8 @@ int maat_remove_group_from_compile(struct maat_compile **compile_hash,
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group %d vtable_id %d from clause %d of compile %d failed, literal is not in compile.",
__FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, g2c_item->compile_id);
__FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index,
g2c_item->compile_id);
return -1;
}
@@ -1181,7 +1183,7 @@ int maat_remove_group_from_compile(struct maat_compile **compile_hash,
static inline int compare_clause_id(const void *a, const void *b)
{
long long ret = *(const unsigned long long *)a - *(const unsigned long long *)b;
long long ret = *(const long long *)a - *(const long long *)b;
if (0 == ret) {
return 0;
@@ -1242,7 +1244,7 @@ static int maat_compile_has_literal(struct maat_compile* compile,
continue;
}
tmp = (struct maat_literal_id*)utarray_find(clause_state->literal_ids,
tmp = (struct maat_literal_id*)utarray_find(clause_state->ut_literal_ids,
literal_id, compare_literal_id);
if (tmp) {
assert(tmp->group_id == literal_id->group_id &&
@@ -1277,7 +1279,9 @@ size_t compile_runtime_get_new_hit_paths(struct compile_runtime *compile_rt,
struct maat_literal_id literal_id = {0, 0};
struct maat_hit_path tmp_path;
struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM;
assert(compile_state->thread_id >= 0);
pthread_rwlock_rdlock(&compile_rt->rwlock);
int bool_match_ret = bool_matcher_match(compile_rt->bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses), expr_match,
@@ -1312,6 +1316,7 @@ size_t compile_runtime_get_new_hit_paths(struct compile_runtime *compile_rt,
}
}
}
pthread_rwlock_unlock(&compile_rt->rwlock);
return new_hit_path_cnt;
}
@@ -1345,7 +1350,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
struct maat_clause_state *clause_state = NULL;
struct maat_literal_id literal_id = {group_id, vtable_id};
struct maat_literal_id *tmp = NULL;
unsigned long long *clause_id = 0;
long long *clause_id = 0;
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
pthread_rwlock_rdlock(&compile_rt->rwlock);
@@ -1359,7 +1364,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
}
size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids,
tmp = (struct maat_literal_id *)utarray_find(clause_state->ut_literal_ids,
&literal_id, compare_literal_id);
if (tmp) {
//Deduplication
@@ -1377,7 +1382,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
utarray_len(compile_state->this_scan_hit_clauses) - new_clause_idx);
for (i = new_clause_idx; i < utarray_len(compile_state->this_scan_hit_clauses); i++) {
clause_id = (unsigned long long *)utarray_eltptr(compile_state->this_scan_hit_clauses, i);
clause_id = (long long *)utarray_eltptr(compile_state->this_scan_hit_clauses, i);
utarray_push_back(compile_state->all_hit_clauses, clause_id);
}
utarray_sort(compile_state->all_hit_clauses, compare_clause_id);
@@ -1539,7 +1544,7 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
} else {
maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile);
}
// printf("compile_runtime_update compile_id:%lu, compile->declared_clause_num:%d\n",
// printf("compile_runtime_update compile_id:%lld, compile->declared_clause_num:%d\n",
// compile->compile_id, compile->declared_clause_num);
pthread_rwlock_unlock(&compile_rt->rwlock);
@@ -1621,23 +1626,17 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name)
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
//TODO: no need add rdlock?
size_t compile_cnt = maat_compile_in_use_count(compile_rt->compile_hash);
if (0 == compile_cnt) {
return 0;
}
struct bool_matcher *old_bool_matcher = NULL;
struct bool_matcher *new_bool_matcher = NULL;
size_t compile_cnt = HASH_COUNT(compile_rt->compile_hash);
log_info(compile_rt->logger, MODULE_COMPILE,
"table[%s] committing %zu compile rules for rebuilding compile bool_matcher engine",
table_name, compile_cnt);
int ret = 0;
new_bool_matcher = maat_compile_bool_matcher_new(compile_rt->compile_hash,
&compile_rt->clause_id_generator,
compile_rt->logger);
new_bool_matcher = maat_compile_bool_matcher_new(compile_rt);
if (NULL == new_bool_matcher) {
log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] table[%s] rebuild compile bool_matcher engine failed when update %zu compile rules",
@@ -1711,12 +1710,8 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
struct compile_rule *compile_rules[compile_ids_size];
// all hit clause_id -> compile_id
pthread_rwlock_rdlock(&compile_rt->rwlock);
size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt->bm,
is_last_scan, compile_state,
(void **)compile_rules,
compile_ids_size);
pthread_rwlock_unlock(&compile_rt->rwlock);
size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt, is_last_scan, compile_state,
(void **)compile_rules, compile_ids_size);
if (bool_match_ret > 0) {
qsort(compile_rules, bool_match_ret, sizeof(struct compile_rule *),
compare_compile_rule);
@@ -1729,7 +1724,7 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
return MIN(bool_match_ret, compile_ids_size);
}
int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id,
long long *hit_item_ids, size_t hit_item_cnt,
size_t *n_hit_group_id, struct maat_state *state)
{
@@ -1745,7 +1740,7 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
}
for (size_t i = 0; i < hit_item_cnt; i++) {
HASH_FIND(hh, item_hash, &(hit_item_ids[i]), sizeof(long long), item);
item = (struct maat_item *)rcu_hash_find(item_htable, (char *)&(hit_item_ids[i]), sizeof(long long));
//assert(item != NULL);
if (!item) {
// item config has been deleted

29
src/maat_config_monitor.c
View File

@@ -109,9 +109,9 @@ char *read_nxt_line_from_buff(const char *buff, size_t buff_size,
return line;
}
int cm_read_table_file(struct cm_table_info_t* index,
int cm_read_table_file(struct cm_table_info_t *index,
int (*update_fn)(const char *, const char *, void *),
void* u_param)
void *u_param, struct log_handle *logger)
{
int cfg_num = 0,i =0;
int ret = 0;
@@ -123,7 +123,8 @@ int cm_read_table_file(struct cm_table_info_t* index,
ret = load_file_to_memory(index->cfg_path, (unsigned char **)&table_file_buff, &file_sz);
if (ret < 0) {
// log_error
log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] open %s failed.",
__FUNCTION__, __LINE__, index->cfg_path);
return -1;
}
@@ -131,7 +132,9 @@ int cm_read_table_file(struct cm_table_info_t* index,
sscanf(line, "%d\n", &cfg_num);
if(cfg_num != index->cfg_num) {
//log_error
FREE(table_file_buff);
log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] file %s config num not matched",
__FUNCTION__, __LINE__, index->cfg_path);
return -1;
}
@@ -140,17 +143,19 @@ int cm_read_table_file(struct cm_table_info_t* index,
ret_str = read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
if (ret_str == NULL) {
//log_error
log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] file %s line_num %d less than claimed %d",
__FUNCTION__, __LINE__, index->cfg_path, i, cfg_num);
break;
}
if(line[sizeof(line) - 1] != '\0') {
//log_error
log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] line size more than %u at of file %s:%d",
__FUNCTION__, __LINE__, sizeof(line), index->cfg_path, i);
continue;
}
ret = update_fn(index->table_name, line, u_param);
if (ret<0) {
if (ret < 0) {
break;
}
}
@@ -220,7 +225,7 @@ int get_new_idx_path(long long current_version, const char *file_dir,
char ***idx_path, size_t *idx_num, struct log_handle *logger)
{
struct dirent **namelist = NULL;
int update_type = MAAT_UPDATE_TYPE_NONE;
int update_type = MAAT_UPDATE_TYPE_INVALID;
int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort);
if (n < 0) {
@@ -297,7 +302,7 @@ int get_new_idx_path(long long current_version, const char *file_dir,
*idx_num = inc_idx_num;
update_type = MAAT_UPDATE_TYPE_INC;
} else {
update_type = MAAT_UPDATE_TYPE_NONE;
update_type = MAAT_UPDATE_TYPE_INVALID;
}
FREE(inc_file_idx);
@@ -324,7 +329,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
memset(table_array, 0, sizeof(table_array));
int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num, logger);
if (update_type != MAAT_UPDATE_TYPE_NONE) {
if (update_type != MAAT_UPDATE_TYPE_INVALID) {
for (i = 0; i < idx_path_num; i++) {
log_info(logger, MODULE_CONFIG_MONITOR, "load %s", idx_path_array[i]);
int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, MAX_TABLE_NUM);
@@ -343,7 +348,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
}
for (int j = 0; j < table_num; j++) {
cm_read_table_file(table_array + j, update_fn, u_param);
cm_read_table_file(table_array + j, update_fn, u_param, logger);
}
if (finish_fn != NULL) {
@@ -424,9 +429,7 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL,
strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL,
maat_instance->logger);
FREE(json_buff);
json_buff = NULL;
if (ret < 0) {
return -1;
}

80
src/maat_ex_data.c
View File

@@ -26,6 +26,7 @@ struct ex_data_runtime {
struct rcu_hash_table *htable;
struct ex_data_schema *ref_ex_schema;
struct maat_garbage_bin *ref_garbage_bin;
int table_id;
struct log_handle *logger;
@@ -33,20 +34,21 @@ struct ex_data_runtime {
void cache_row_free(void *p)
{
FREE(*(char **)p);
free(*(char **)p);
}
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
struct ex_data_runtime *
ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn,
struct log_handle *logger)
struct maat_garbage_bin *garbage_bin, struct log_handle *logger)
{
struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1);
utarray_new(ex_data_rt->cache_rows, &ut_cache_row_icd);
ex_data_rt->htable = rcu_hash_new(data_free_fn);
ex_data_rt->table_id = table_id;
ex_data_rt->ref_garbage_bin = garbage_bin;
ex_data_rt->logger = logger;
return ex_data_rt;
@@ -137,7 +139,7 @@ struct ex_data_schema *ex_data_schema_new(maat_ex_new_func_t *new_func,
void ex_data_schema_free(struct ex_data_schema *ex_schema)
{
FREE(ex_schema);
free(ex_schema);
}
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt,
@@ -146,16 +148,16 @@ void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt,
ex_data_rt->ref_ex_schema = schema;
}
void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt,
struct ex_container_ctx *container_ctx)
void ex_data_runtime_set_ex_container_schema(struct ex_data_runtime *ex_data_rt,
struct ex_container_schema *container_schema)
{
rcu_hash_set_user_ctx(ex_data_rt->htable, container_ctx);
rcu_hash_set_user_ctx(ex_data_rt->htable, container_schema);
}
struct ex_container_ctx *
ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt)
struct ex_container_schema *
ex_data_runtime_get_ex_container_schema(struct ex_data_runtime *ex_data_rt)
{
return (struct ex_container_ctx *)rcu_hash_get_user_ctx(ex_data_rt->htable);
return (struct ex_container_schema *)rcu_hash_get_user_ctx(ex_data_rt->htable);
}
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row,
@@ -169,9 +171,9 @@ void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char
return ex_data;
}
struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data)
struct ex_container *ex_container_new(void *ex_data, void *custom_data)
{
struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1);
struct ex_container *ex_container = ALLOC(struct ex_container, 1);
ex_container->ex_data = ex_data;
ex_container->custom_data = custom_data;
@@ -179,26 +181,31 @@ struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data
return ex_container;
}
void ex_data_container_free(void *ctx, void *data)
void ex_container_free(void *schema, void *data)
{
if (NULL == ctx || NULL == data) {
/* schema is NULL if not call ex_data_runtime_set_ex_container_schema */
if (NULL == data) {
return;
}
//TODO:
if (NULL == schema) {
struct ex_container_ctx *container_ctx = (struct ex_container_ctx *)ctx;
long argl = container_ctx->ex_schema->argl;
void *argp = container_ctx->ex_schema->argp;
}
struct ex_data_container *ex_container = (struct ex_data_container *)data;
struct ex_container_schema *container_schema = (struct ex_container_schema *)schema;
long argl = container_schema->ex_schema->argl;
void *argp = container_schema->ex_schema->argp;
struct ex_container *ex_container = (struct ex_container *)data;
if (ex_container->ex_data != NULL
&& container_ctx->ex_schema->free_func != NULL) {
container_ctx->ex_schema->free_func(container_ctx->table_id,
&& container_schema->ex_schema->free_func != NULL) {
container_schema->ex_schema->free_func(container_schema->table_id,
&(ex_container->ex_data), argl, argp);
}
if (ex_container->custom_data != NULL
&& container_ctx->custom_data_free != NULL) {
container_ctx->custom_data_free(ex_container->custom_data);
&& container_schema->custom_data_free != NULL) {
container_schema->custom_data_free(ex_container->custom_data);
}
FREE(ex_container);
@@ -206,11 +213,11 @@ void ex_data_container_free(void *ctx, void *data)
int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len,
struct ex_data_container *ex_container)
struct ex_container *ex_container)
{
struct ex_data_container *tmp_container = NULL;
struct ex_container *tmp_container = NULL;
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
tmp_container = (struct ex_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (tmp_container != NULL) {
log_error(ex_data_rt->logger, MODULE_EX_DATA,
@@ -227,8 +234,8 @@ int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{
struct ex_data_container *tmp_container = NULL;
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
struct ex_container *tmp_container = NULL;
tmp_container = (struct ex_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (NULL == tmp_container) {
log_error(ex_data_rt->logger, MODULE_EX_DATA,
@@ -245,9 +252,9 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{
struct ex_data_container *ex_container = NULL;
struct ex_container *ex_container = NULL;
ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
ex_container = (struct ex_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (NULL == ex_container) {
return NULL;
@@ -262,7 +269,7 @@ void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt,
}
void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container *ex_container)
struct ex_container *ex_container)
{
void *dup_ex_data = NULL;
ex_data_rt->ref_ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data,
@@ -275,8 +282,8 @@ void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_r
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{
struct ex_data_container *ex_container = NULL;
ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
struct ex_container *ex_container = NULL;
ex_container = (struct ex_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (NULL == ex_container) {
return NULL;
@@ -290,8 +297,13 @@ size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt)
return rcu_hash_count(ex_data_rt->htable);
}
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container ***ex_container)
int ex_data_runtime_is_updating(struct ex_data_runtime *ex_data_rt)
{
return rcu_hash_list_updating_data(ex_data_rt->htable, (void ***)ex_container);
return rcu_hash_is_updating(ex_data_rt->htable);
}
size_t ex_data_runtime_list_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_container ***ex_container)
{
return rcu_hash_list(ex_data_rt->htable, (void ***)ex_container);
}

150
src/maat_expr.c
View File

@@ -60,7 +60,7 @@ enum match_method {
struct expr_item {
long long item_id;
long long group_id;
int district_id;
long long district_id;
char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type;
enum hs_match_mode match_mode;
@@ -73,14 +73,11 @@ struct expr_runtime {
enum hs_pattern_type pattern_type;
struct adapter_hs *hs;
struct adapter_hs_stream *hs_stream;
struct rcu_hash_table *htable;
struct rcu_hash_table *htable; // store hs_expr rule for rebuild adapter_hs instance
struct rcu_hash_table *item_htable; // store this expr table's all maat_item which will be used in expr_runtime_scan
struct group2group_runtime *ref_g2g_rt;
uint32_t rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
int n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
@@ -401,7 +398,7 @@ void expr_schema_free(void *expr_schema)
FREE(expr_schema);
}
void expr_rule_free(and_expr_t *expr_rule)
void expr_rule_free(struct hs_expr *expr_rule)
{
if (NULL == expr_rule) {
return;
@@ -416,7 +413,7 @@ void expr_rule_free(and_expr_t *expr_rule)
void expr_ex_data_free(void *user_ctx, void *data)
{
and_expr_t *expr_rule = (and_expr_t *)data;
struct hs_expr *expr_rule = (struct hs_expr *)data;
expr_rule_free(expr_rule);
}
@@ -432,9 +429,9 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num,
struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1);
expr_rt->htable = rcu_hash_new(expr_ex_data_free);
expr_rt->item_htable = rcu_hash_new(maat_item_free);
expr_rt->scan_mode = schema->scan_mode;
expr_rt->pattern_type = schema->pattern_type;
expr_rt->item_user_data_free = maat_item_inner_free;
expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger;
@@ -467,10 +464,9 @@ void expr_runtime_free(void *expr_runtime)
expr_rt->htable = NULL;
}
struct maat_item *item = NULL, *tmp = NULL;
HASH_ITER(hh, expr_rt->item_hash, item, tmp) {
HASH_DELETE(hh, expr_rt->item_hash, item);
maat_item_free(item, expr_rt->item_user_data_free);
if (expr_rt->item_htable != NULL) {
rcu_hash_free(expr_rt->item_htable);
expr_rt->item_htable = NULL;
}
if (expr_rt->hit_cnt != NULL) {
@@ -486,9 +482,8 @@ void expr_runtime_free(void *expr_runtime)
FREE(expr_rt);
}
int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
size_t key_len, long long item_id, and_expr_t *expr_rule,
int is_valid, struct log_handle *logger)
int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len,
long long item_id, struct hs_expr *expr_rule, int is_valid)
{
void *data = NULL;
@@ -496,21 +491,34 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
//delete
data = rcu_hash_find(expr_rt->htable, key, key_len);
if (NULL == data) {
log_error(logger, MODULE_EXPR,
"[%s:%d] the key of expr rule not exist, can't be deleted, expr_id:%llu",
data = rcu_hash_updating_find(expr_rt->htable, key, key_len);
if (NULL == data) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] the key of expr rule(rule_id:%lld) not exist, can't be deleted from expr runtime htable",
__FUNCTION__, __LINE__, item_id);
return -1;
}
}
rcu_hash_del(expr_rt->htable, key, key_len);
} else {
//add
data = rcu_hash_find(expr_rt->htable, key, key_len);
if (data != NULL) {
log_error(logger, MODULE_EXPR,
"[%s:%d] the key of expr rule already exist, can't be added, expr_id:%llu",
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] the key of expr rule(rule_id:%lld) already exist, can't be added to expr runtime htable",
__FUNCTION__, __LINE__, item_id);
return -1;
}
data = rcu_hash_updating_find(expr_rt->htable, key, key_len);
if (data != NULL) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] the key of expr rule(rule_id:%lld) already exist, can't be added to expr runtime htable",
__FUNCTION__, __LINE__, item_id);
return -1;
}
rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule);
}
@@ -565,7 +573,7 @@ static size_t hex2bin(char *hex, int hex_len, char *binary, size_t size)
}
#define MAAT_MAX_EXPR_ITEM_NUM 8
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data,
struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data,
struct log_handle *logger)
{
size_t i = 0;
@@ -581,7 +589,7 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data,
memset(key_left_offset, -1, sizeof(key_left_offset));
memset(key_right_offset, -1, sizeof(key_right_offset));
and_expr_t *expr_rule = ALLOC(and_expr_t, 1);
struct hs_expr *expr_rule = ALLOC(struct hs_expr, 1);
switch (expr_item->expr_type) {
case EXPR_TYPE_AND:
@@ -718,7 +726,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
int ret = -1;
struct maat_item_inner *u_para = NULL;
struct maat_item *item = NULL;
and_expr_t *expr_rule = NULL;
struct hs_expr *expr_rule = NULL;
struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
@@ -728,30 +736,34 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
}
int is_valid = get_column_value(line, valid_column);
//printf("<expr_runtime_update> item_id:%lld is_valid:%d\n", item_id, is_valid);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND(hh, expr_rt->item_hash, &item_id, sizeof(long long), item);
item = (struct maat_item *)rcu_hash_find(expr_rt->item_htable, (char *)&item_id, sizeof(item_id));
if (NULL == item) {
item = (struct maat_item *)rcu_hash_updating_find(expr_rt->item_htable, (char *)&item_id, sizeof(item_id));
if (NULL == item) {
return -1;
}
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
rcu_hash_del(expr_rt->item_htable, (char *)&item_id, sizeof(item_id));
} else {
//add
item = (struct maat_item *)rcu_hash_find(expr_rt->item_htable, (char *)&item_id, sizeof(item_id));
if (item) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr runtime add item(item_id:%lld) to item_htable failed, already exist",
__FUNCTION__, __LINE__, item_id);
return -1;
}
HASH_DELETE(hh, expr_rt->item_hash, item);
maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else {
//add
HASH_FIND(hh, expr_rt->item_hash, &item_id, sizeof(long long), item);
item = (struct maat_item *)rcu_hash_updating_find(expr_rt->item_htable, (char *)&item_id, sizeof(item_id));
if (item) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr runtime add item %llu to item_hash failed, already exist",
"[%s:%d] expr runtime add item(item_id:%lld) to item_htable failed, already exist",
__FUNCTION__, __LINE__, item_id);
return -1;
}
@@ -762,33 +774,27 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
}
u_para = maat_item_inner_new(expr_item->group_id, item_id, expr_item->district_id);
item = maat_item_new(item_id, expr_item->group_id, u_para);
HASH_ADD(hh, expr_rt->item_hash, item_id, sizeof(long long), item);
item = maat_item_new(item_id, expr_item->group_id, u_para, maat_item_inner_free);
rcu_hash_add(expr_rt->item_htable, (char *)&item_id, sizeof(item_id), item);
expr_rule = expr_item_to_expr_rule(expr_item, u_para, expr_rt->logger);
expr_item_free(expr_item);
if (NULL == expr_rule) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] transform expr table(table_id:%d) item to expr_rule failed, item_id:%llu",
"[%s:%d] transform expr table(table_id:%d) item(item_id:%lld) to expr_rule failed",
__FUNCTION__, __LINE__, schema->table_id, item_id);
return -1;
}
}
char *key = (char *)&item_id;
ret = expr_runtime_update_row(expr_rt, key, sizeof(long long), item_id,
expr_rule, is_valid, expr_rt->logger);
ret = expr_runtime_update_row(expr_rt, key, sizeof(item_id), item_id,
expr_rule, is_valid);
if (ret < 0) {
if (expr_rule != NULL) {
expr_rule_free(expr_rule);
}
return -1;
} else {
if (0 == is_valid) {
expr_rt->rule_num--;
} else {
expr_rt->rule_num++;
}
}
return 0;
@@ -800,30 +806,39 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
return -1;
}
int ret = 0;
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array);
if (0 == rule_cnt) {
FREE(ex_data_array);
int updating_flag = rcu_hash_is_updating(expr_rt->htable);
if (0 == updating_flag) {
return 0;
}
and_expr_t *rules = ALLOC(and_expr_t, rule_cnt);
for (size_t i = 0 ; i < rule_cnt; i++) {
rules[i] = *(and_expr_t *)ex_data_array[i];
}
rcu_hash_commit(expr_rt->htable);
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
struct hs_expr *rules = NULL;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list(expr_rt->htable, &ex_data_array);
//printf("rcu_hash_commit rule_cnt:%zu\n", rule_cnt);
if (rule_cnt > 0) {
rules = ALLOC(struct hs_expr, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct hs_expr *)ex_data_array[i];
//printf("expr_id:%lld\n", rules[i].expr_id);
// for (size_t j = 0; j < rules[i].n_patterns; j++) {
// printf("rules[%zu].patterns[%zu]:%s\n", i, j, rules[i].patterns[j].pat);
// }
}
}
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] committing %zu expr rules for rebuilding adapter_hs engine",
table_name, rule_cnt);
int ret = 0;
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, expr_rt->pattern_type,
expr_rt->n_worker_thread,
rules, rule_cnt,
expr_rt->n_worker_thread, rules, rule_cnt,
expr_rt->logger);
if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR,
@@ -834,15 +849,20 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs;
if (old_adapter_hs != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs,
(void (*)(void*))adapter_hs_destroy);
rcu_hash_commit(expr_rt->htable);
rule_cnt = rcu_hash_updating_count(expr_rt->htable);
assert(rule_cnt == 0);
}
rcu_hash_commit(expr_rt->item_htable);
expr_rt->rule_num = rule_cnt;
if (rules != NULL) {
FREE(rules);
}
if (ex_data_array != NULL) {
FREE(ex_data_array);
}
return ret;
}
@@ -879,7 +899,7 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
int district_id = state->district_id;
long long district_id = state->district_id;
memset(hit_item_ids, 0, sizeof(hit_item_ids));
@@ -891,7 +911,7 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
}
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids,
ret = maat_compile_state_update(expr_rt->item_htable, vtable_id, hit_item_ids,
real_hit_item_cnt, &group_hit_cnt, state);
if (ret < 0) {
return -1;
@@ -952,7 +972,7 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, siz
}
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids,
ret = maat_compile_state_update(expr_rt->item_htable, vtable_id, hit_item_ids,
n_hit_item, &group_hit_cnt, state);
if (ret < 0) {
return -1;

111
src/maat_flag.c
View File

@@ -37,20 +37,16 @@ struct flag_schema {
struct flag_item {
long long item_id;
long long group_id;
int district_id;
long long district_id;
long long flag;
long long flag_mask;
};
struct flag_runtime {
struct flag_matcher *matcher;
struct rcu_hash_table *htable;
struct rcu_hash_table *htable; //store flag rule for rebuild flag_matcher instance
struct rcu_hash_table *item_htable; //store this flag table's all maat_item which will be used in flag_runtime_scan
uint32_t rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
@@ -143,7 +139,7 @@ void *flag_runtime_new(void *flag_schema, int max_thread_num,
struct flag_runtime *flag_rt = ALLOC(struct flag_runtime, 1);
flag_rt->htable = rcu_hash_new(flag_ex_data_free);
flag_rt->item_user_data_free = maat_item_inner_free;
flag_rt->item_htable = rcu_hash_new(maat_item_free);
flag_rt->ref_garbage_bin = garbage_bin;
flag_rt->logger = logger;
@@ -165,10 +161,14 @@ void flag_runtime_free(void *flag_runtime)
flag_rt->htable = NULL;
}
struct maat_item *item = NULL, *tmp = NULL;
HASH_ITER(hh, flag_rt->item_hash, item, tmp) {
HASH_DELETE(hh, flag_rt->item_hash, item);
maat_item_free(item, flag_rt->item_user_data_free);
if (flag_rt->item_htable != NULL) {
rcu_hash_free(flag_rt->item_htable);
flag_rt->item_htable = NULL;
}
if (flag_rt->matcher != NULL) {
flag_matcher_free(flag_rt->matcher);
flag_rt->matcher = NULL;
}
if (flag_rt->hit_cnt != NULL) {
@@ -192,22 +192,35 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_
if (0 == is_valid) {
//delete
data = rcu_hash_find(flag_rt->htable, key, key_len);
if (NULL == data) {
data = rcu_hash_updating_find(flag_rt->htable, key, key_len);
if (NULL == data) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] the key of flag rule not exist, can't be deleted, item_id:%llu",
"[%s:%d] the key of flag rule(rule_id:%lld) not exist, can't be deleted from flag runtime htable",
__FUNCTION__, __LINE__, item_id);
return -1;
}
}
rcu_hash_del(flag_rt->htable, key, key_len);
} else {
//add
data = rcu_hash_find(flag_rt->htable, key, key_len);
if (data != NULL) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] the key of flag rule already exist, can't be added, item_id:%llu",
"[%s:%d] the key of flag rule(rule_id:%lld) already exist, can't be added to flag runtime htable",
__FUNCTION__, __LINE__, item_id);
return -1;
}
data = rcu_hash_updating_find(flag_rt->htable, key, key_len);
if (data != NULL) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] the key of flag rule(rule_id:%lld) already exist, can't be added to flag runtime htable",
__FUNCTION__, __LINE__, item_id);
return -1;
}
rcu_hash_add(flag_rt->htable, key, key_len, (void *)rule);
}
@@ -334,26 +347,29 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(long long), item);
item = (struct maat_item *)rcu_hash_find(flag_rt->item_htable, (char *)&item_id, sizeof(item_id));
if (NULL == item) {
item = (struct maat_item *)rcu_hash_updating_find(flag_rt->item_htable, (char *)&item_id, sizeof(item_id));
if (NULL == item) {
return -1;
}
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
rcu_hash_del(flag_rt->item_htable, (char *)&item_id, sizeof(item_id));
} else {
//add
item = (struct maat_item *)rcu_hash_find(flag_rt->item_htable, (char *)&item_id, sizeof(item_id));
if (item) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag runtime add item(item_id:%lld) to item_htable failed, already exist",
__FUNCTION__, __LINE__, item_id);
return -1;
}
HASH_DELETE(hh, flag_rt->item_hash, item);
maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else {
//add
HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(long long), item);
item = (struct maat_item *)rcu_hash_updating_find(flag_rt->item_htable, (char *)&item_id, sizeof(item_id));
if (item) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag runtime add item %d to item_hash failed, already exist",
"[%s:%d] flag runtime add item(item_id:%lld) to item_htable failed, already exist",
__FUNCTION__, __LINE__, item_id);
return -1;
}
@@ -364,14 +380,14 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
}
u_para = maat_item_inner_new(flag_item->group_id, item_id, flag_item->district_id);
item = maat_item_new(item_id, flag_item->group_id, u_para);
HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(long long), item);
item = maat_item_new(item_id, flag_item->group_id, u_para, maat_item_inner_free);
rcu_hash_add(flag_rt->item_htable, (char *)&(item_id), sizeof(item_id), item);
flag_rule = flag_item_to_flag_rule(flag_item, u_para);
flag_item_free(flag_item);
if (NULL == flag_rule) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] transform flag table(table_id:%d) item to flag_rule failed, item_id:%d",
"[%s:%d] transform flag table(table_id:%d) item(item_id:%lld) to flag_rule failed",
__FUNCTION__, __LINE__, schema->table_id, item_id);
return -1;
}
@@ -402,28 +418,32 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name)
return -1;
}
int ret = 0;
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list_updating_data(flag_rt->htable, &ex_data_array);
if (0 == rule_cnt) {
FREE(ex_data_array);
int updating_flag = rcu_hash_is_updating(flag_rt->htable);
if (0 == updating_flag) {
return 0;
}
struct flag_rule *rules = ALLOC(struct flag_rule, rule_cnt);
rcu_hash_commit(flag_rt->htable);
struct flag_rule *rules = NULL;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list(flag_rt->htable, &ex_data_array);
if (rule_cnt > 0) {
rules = ALLOC(struct flag_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct flag_rule *)ex_data_array[i];
}
struct flag_matcher *new_flag_matcher = NULL;
struct flag_matcher *old_flag_matcher = NULL;
}
log_info(flag_rt->logger, MODULE_FLAG,
"table[%s] committing %zu flag rules for rebuilding flag_matcher engine",
table_name, rule_cnt);
int ret = 0;
struct flag_matcher *new_flag_matcher = NULL;
struct flag_matcher *old_flag_matcher = NULL;
new_flag_matcher = flag_matcher_new(rules, rule_cnt);
if (NULL == new_flag_matcher) {
log_error(flag_rt->logger, MODULE_FLAG,
@@ -434,15 +454,20 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name)
old_flag_matcher = flag_rt->matcher;
flag_rt->matcher = new_flag_matcher;
if (old_flag_matcher != NULL) {
maat_garbage_bagging(flag_rt->ref_garbage_bin, old_flag_matcher,
(void (*)(void*))flag_matcher_free);
rcu_hash_commit(flag_rt->htable);
rule_cnt = rcu_hash_updating_count(flag_rt->htable);
assert(rule_cnt == 0);
}
rcu_hash_commit(flag_rt->item_htable);
flag_rt->rule_num = rule_cnt;
if (rules != NULL) {
FREE(rules);
}
if (ex_data_array != NULL) {
FREE(ex_data_array);
}
return ret;
}
@@ -473,7 +498,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
int district_id = state->district_id;
long long district_id = state->district_id;
memset(hit_item_ids, 0, sizeof(hit_item_ids));
@@ -485,7 +510,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
}
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids,
int ret = maat_compile_state_update(flag_rt->item_htable, vtable_id, hit_item_ids,
n_hit_item, &group_hit_cnt, state);
if (ret < 0) {
return -1;

45
src/maat_fqdn_plugin.c
View File

@@ -112,7 +112,7 @@ void fqdn_plugin_schema_free(void *fqdn_plugin_schema)
schema->ex_schema = NULL;
}
FREE(schema);
free(schema);
}
/* fqdn plugin table ex data API */
@@ -162,8 +162,8 @@ void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, int max_thread_num,
struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema;
struct fqdn_plugin_runtime *fqdn_plugin_rt = ALLOC(struct fqdn_plugin_runtime, 1);
fqdn_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free,
logger);
fqdn_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_container_free,
garbage_bin, logger);
fqdn_plugin_rt->ref_garbage_bin = garbage_bin;
fqdn_plugin_rt->logger = logger;
@@ -319,7 +319,7 @@ int fqdn_plugin_runtime_update_row(struct fqdn_plugin_runtime *fqdn_plugin_rt,
} else {
// add
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)fqdn_plugin_rule);
struct ex_container *ex_container = ex_container_new(ex_data, (void *)fqdn_plugin_rule);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
return -1;
@@ -389,32 +389,39 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
return -1;
}
int ret = 0;
struct ex_data_container **ex_container = NULL;
struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime;
struct ex_data_runtime *ex_data_rt = fqdn_plugin_rt->ex_data_rt;
if (NULL == ex_data_rt) {
return -1;
}
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (0 == rule_cnt) {
FREE(ex_container);
int updating_flag = ex_data_runtime_is_updating(ex_data_rt);
if (0 == updating_flag) {
return 0;
}
struct FQDN_rule *rules = ALLOC(struct FQDN_rule, rule_cnt);
ex_data_runtime_commit(ex_data_rt);
struct FQDN_rule *rules = NULL;
struct ex_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
if (rule_cnt > 0) {
rules = ALLOC(struct FQDN_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct FQDN_rule *)ex_container[i]->custom_data;
assert(rules[i].user_tag == ex_container[i] || rules[i].user_tag == NULL);
rules[i].user_tag = ex_container[i];
}
struct FQDN_engine *new_fqdn_engine = NULL;
struct FQDN_engine *old_fqdn_engine = NULL;
}
log_info(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
"table[%s] committing %zu fqdn_plugin rules for rebuilding FQDN engine",
table_name, rule_cnt);
int ret = 0;
struct FQDN_engine *new_fqdn_engine = NULL;
struct FQDN_engine *old_fqdn_engine = NULL;
new_fqdn_engine = FQDN_engine_new(rules, rule_cnt);
if (NULL == new_fqdn_engine) {
log_error(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
@@ -425,12 +432,20 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
old_fqdn_engine = fqdn_plugin_rt->engine;
fqdn_plugin_rt->engine = new_fqdn_engine;
if (old_fqdn_engine != NULL) {
maat_garbage_bagging(fqdn_plugin_rt->ref_garbage_bin, old_fqdn_engine,
(void (*)(void*))FQDN_engine_free);
ex_data_runtime_commit(ex_data_rt);
}
fqdn_plugin_rt->rule_num = rule_cnt;
if (rules != NULL) {
FREE(rules);
}
if (ex_container != NULL) {
FREE(ex_container);
}
return ret;
}
@@ -466,7 +481,7 @@ int fqdn_plugin_runtime_get_ex_data(void *fqdn_plugin_runtime, const char *query
int n_result = FQDN_engine_search(fqdn_plugin_rt->engine, query_fqdn, strlen(query_fqdn), results, n_ex_data);
for (int i = 0; i < n_result; i++) {
ex_data_array[i] = ex_data_runtime_get_ex_data_by_container(fqdn_plugin_rt->ex_data_rt,
(struct ex_data_container *)results[i].user_tag);
(struct ex_container *)results[i].user_tag);
}
return n_result;

11
src/maat_group.c
View File

@@ -149,8 +149,8 @@ void *group2group_runtime_new(void *g2g_schema, int max_thread_num,
void group_vertex_free(struct maat_group *group)
{
free(group->top_group_ids);
free(group);
FREE(group->top_group_ids);
FREE(group);
}
void maat_group_topology_free(struct maat_group_topology *group_topo)
@@ -165,6 +165,7 @@ void maat_group_topology_free(struct maat_group_topology *group_topo)
assert(group_topo->hash_group_by_id == NULL);
igraph_destroy(&group_topo->group_graph);
FREE(group_topo);
}
void group2group_runtime_free(void *g2g_runtime)
@@ -175,12 +176,14 @@ void group2group_runtime_free(void *g2g_runtime)
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
pthread_rwlock_wrlock(&g2g_rt->rwlock);
if (g2g_rt->group_topo != NULL) {
maat_group_topology_free(g2g_rt->group_topo);
g2g_rt->group_topo = NULL;
}
//pthread_rwlock_unlock(&g2g_rt->rwlock);
pthread_rwlock_unlock(&g2g_rt->rwlock);
pthread_rwlock_destroy(&g2g_rt->rwlock);
FREE(g2g_rt);
}

121
src/maat_interval.c
View File

@@ -34,20 +34,16 @@ struct interval_schema {
struct interval_item {
long long item_id;
long long group_id;
int district_id;
long long district_id;
int low_bound;
int up_bound;
};
struct interval_runtime {
struct interval_matcher *matcher;
struct rcu_hash_table *htable;
struct rcu_hash_table *htable; //store interval rule for rebuild interval_matcher instance
struct rcu_hash_table *item_htable; //store this interval table's all maat_item which will be used in interval_runtime_scan
uint32_t rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
@@ -140,7 +136,7 @@ void *interval_runtime_new(void *interval_schema, int max_thread_num,
struct interval_runtime *interval_rt = ALLOC(struct interval_runtime, 1);
interval_rt->htable = rcu_hash_new(interval_ex_data_free);
interval_rt->item_user_data_free = maat_item_inner_free;
interval_rt->item_htable = rcu_hash_new(maat_item_free);
interval_rt->ref_garbage_bin = garbage_bin;
interval_rt->logger = logger;
@@ -162,10 +158,14 @@ void interval_runtime_free(void *interval_runtime)
interval_rt->htable = NULL;
}
struct maat_item *item = NULL, *tmp = NULL;
HASH_ITER(hh, interval_rt->item_hash, item, tmp) {
HASH_DELETE(hh, interval_rt->item_hash, item);
maat_item_free(item, interval_rt->item_user_data_free);
if (interval_rt->item_htable != NULL) {
rcu_hash_free(interval_rt->item_htable);
interval_rt->item_htable = NULL;
}
if (interval_rt->matcher != NULL) {
interval_matcher_free(interval_rt->matcher);
interval_rt->matcher = NULL;
}
if (interval_rt->hit_cnt != NULL) {
@@ -284,22 +284,35 @@ int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key,
if (0 == is_valid) {
//delete
data = rcu_hash_find(interval_rt->htable, key, key_len);
if (NULL == data) {
data = rcu_hash_updating_find(interval_rt->htable, key, key_len);
if (NULL == data) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] the key of interval rule not exist, can't be deleted, item_id:%llu",
"[%s:%d] the key of interval rule(rule_id:%lld) not exist, can't be deleted from interval runtime htable",
__FUNCTION__, __LINE__, item_id);
return -1;
}
}
rcu_hash_del(interval_rt->htable, key, key_len);
} else {
//add
data = rcu_hash_find(interval_rt->htable, key, key_len);
if (data != NULL) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] the key of interval rule already exist, can't be added, item_id:%llu",
"[%s:%d] the key of interval rule(rule_id:%lld) already exist, can't be added to interval runtime htable",
__FUNCTION__, __LINE__, item_id);
return -1;
}
data = rcu_hash_updating_find(interval_rt->htable, key, key_len);
if (data != NULL) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] the key of interval rule(rule_id:%lld) already exist, can't be added to interval runtime htable",
__FUNCTION__, __LINE__, item_id);
return -1;
}
rcu_hash_add(interval_rt->htable, key, key_len, (void *)rule);
}
@@ -330,26 +343,33 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND(hh, interval_rt->item_hash, &item_id, sizeof(long long), item);
item = (struct maat_item *)rcu_hash_find(interval_rt->item_htable,
(char *)&item_id, sizeof(item_id));
if (NULL == item) {
item = (struct maat_item *)rcu_hash_updating_find(interval_rt->item_htable,
(char *)&item_id, sizeof(item_id));
if (NULL == item) {
return -1;
}
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
rcu_hash_del(interval_rt->item_htable, (char *)&item_id, sizeof(item_id));
} else {
//add
item = (struct maat_item *)rcu_hash_find(interval_rt->item_htable,
(char *)&item_id, sizeof(item_id));;
if (item) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval runtime add item(item_id:%lld) to item_htable failed, already exist",
__FUNCTION__, __LINE__, item_id);
return -1;
}
HASH_DELETE(hh, interval_rt->item_hash, item);
maat_garbage_bagging(interval_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else {
//add
HASH_FIND(hh, interval_rt->item_hash, &item_id, sizeof(long long), item);
item = (struct maat_item *)rcu_hash_updating_find(interval_rt->item_htable,
(char *)&item_id, sizeof(item_id));;
if (item) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval runtime add item %llu to item_hash failed, already exist",
"[%s:%d] interval runtime add item(item_id:%lld) to item_htable failed, already exist",
__FUNCTION__, __LINE__, item_id);
return -1;
}
@@ -360,14 +380,14 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
}
u_para = maat_item_inner_new(interval_item->group_id, item_id, interval_item->district_id);
item = maat_item_new(item_id, interval_item->group_id, u_para);
HASH_ADD(hh, interval_rt->item_hash, item_id, sizeof(long long), item);
item = maat_item_new(item_id, interval_item->group_id, u_para, maat_item_inner_free);
rcu_hash_add(interval_rt->item_htable, (char *)&(item_id), sizeof(item_id), item);
interval_rule = interval_item_to_interval_rule(interval_item, u_para);
interval_item_free(interval_item);
if (NULL == interval_rule) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] transform interval table(table_id:%d) item to interval_rule failed, item_id:%llu",
"[%s:%d] transform interval table(table_id:%d) item(item_id:%lld) to interval_rule failed",
__FUNCTION__, __LINE__, schema->table_id, item_id);
return -1;
}
@@ -381,12 +401,6 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
interval_rule = NULL;
}
return -1;
} else {
if (0 == is_valid) {
interval_rt->rule_num--;
} else {
interval_rt->rule_num++;
}
}
return 0;
@@ -398,29 +412,33 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name)
return -1;
}
int ret = 0;
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list_updating_data(interval_rt->htable, &ex_data_array);
if (0 == rule_cnt) {
FREE(ex_data_array);
int updating_flag = rcu_hash_is_updating(interval_rt->htable);
if (0 == updating_flag) {
return 0;
}
struct interval_rule *rules = ALLOC(struct interval_rule, rule_cnt);
rcu_hash_commit(interval_rt->htable);
void **ex_data_array = NULL;
struct interval_rule *rules = NULL;
size_t rule_cnt = rcu_hash_list(interval_rt->htable, &ex_data_array);
if (rule_cnt > 0) {
rules = ALLOC(struct interval_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct interval_rule *)ex_data_array[i];
}
struct interval_matcher *new_interval_matcher = NULL;
struct interval_matcher *old_interval_matcher = NULL;
}
log_info(interval_rt->logger, MODULE_INTERVAL,
"table[%s] committing %zu interval rules for rebuilding interval_matcher engine",
table_name, rule_cnt);
int ret = 0;
struct interval_matcher *new_interval_matcher = NULL;
struct interval_matcher *old_interval_matcher = NULL;
new_interval_matcher = interval_matcher_new(rules, rule_cnt);
if (NULL == new_interval_matcher) {
log_error(interval_rt->logger, MODULE_INTERVAL,
@@ -431,15 +449,20 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name)
old_interval_matcher = interval_rt->matcher;
interval_rt->matcher = new_interval_matcher;
if (old_interval_matcher != NULL) {
maat_garbage_bagging(interval_rt->ref_garbage_bin, old_interval_matcher,
(void (*)(void*))interval_matcher_free);
rcu_hash_commit(interval_rt->htable);
rule_cnt = rcu_hash_updating_count(interval_rt->htable);
assert(rule_cnt == 0);
}
rcu_hash_commit(interval_rt->item_htable);
interval_rt->rule_num = rule_cnt;
if (rules != NULL) {
FREE(rules);
}
if (ex_data_array != NULL) {
FREE(ex_data_array);
}
return ret;
}
@@ -470,7 +493,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
int district_id = state->district_id;
long long district_id = state->district_id;
memset(hit_item_ids, 0, sizeof(hit_item_ids));
@@ -482,7 +505,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
}
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids,
int ret = maat_compile_state_update(interval_rt->item_htable, vtable_id, hit_item_ids,
n_hit_item, &group_hit_cnt, state);
if (ret < 0) {
return -1;

250
src/maat_ip.c
View File

@@ -56,13 +56,9 @@ struct ip_item {
struct ip_runtime {
struct ip_matcher* ip_matcher;
struct ex_data_runtime* ex_data_rt;
struct rcu_hash_table* htable; //store ip rule for rebuild ip_matcher instance
struct rcu_hash_table *item_htable; //store this ip table's all maat_item which will be used in ip_runtime_scan
uint32_t rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
@@ -144,64 +140,6 @@ void ip_schema_free(void *ip_schema)
FREE(ip_schema);
}
void *ip_runtime_new(void *ip_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == ip_schema) {
return NULL;
}
struct ip_schema *schema = (struct ip_schema *)ip_schema;
struct ip_runtime *ip_rt = ALLOC(struct ip_runtime, 1);
ip_rt->ex_data_rt = ex_data_runtime_new(schema->table_id,
ex_data_container_free,
logger);
ip_rt->item_user_data_free = maat_item_inner_free;
ip_rt->ref_garbage_bin = garbage_bin;
ip_rt->logger = logger;
ip_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
ip_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
return ip_rt;
}
void ip_runtime_free(void *ip_runtime)
{
if (NULL == ip_runtime) {
return;
}
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
if (ip_rt->ip_matcher != NULL) {
ip_matcher_free(ip_rt->ip_matcher);
}
if (ip_rt->ex_data_rt != NULL) {
ex_data_runtime_free(ip_rt->ex_data_rt);
}
struct maat_item *item = NULL, *tmp_item = NULL;
HASH_ITER(hh, ip_rt->item_hash, item, tmp_item) {
HASH_DELETE(hh, ip_rt->item_hash, item);
maat_item_free(item, ip_rt->item_user_data_free);
}
if (ip_rt->hit_cnt != NULL) {
alignment_int64_array_free(ip_rt->hit_cnt);
ip_rt->hit_cnt = NULL;
}
if (ip_rt->scan_cnt != NULL) {
alignment_int64_array_free(ip_rt->scan_cnt);
ip_rt->scan_cnt = NULL;
}
FREE(ip_rt);
}
struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema,
struct log_handle *logger)
{
@@ -311,6 +249,64 @@ void ip_item_free(struct ip_item *ip_item)
FREE(ip_item);
}
void ip_ex_data_free(void *user_ctx, void *data)
{
struct ip_item *item = (struct ip_item *)data;
ip_item_free(item);
}
void *ip_runtime_new(void *ip_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
struct ip_runtime *ip_rt = ALLOC(struct ip_runtime, 1);
ip_rt->htable = rcu_hash_new(ip_ex_data_free);
ip_rt->item_htable = rcu_hash_new(maat_item_free);
ip_rt->ref_garbage_bin = garbage_bin;
ip_rt->logger = logger;
ip_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
ip_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
return ip_rt;
}
void ip_runtime_free(void *ip_runtime)
{
if (NULL == ip_runtime) {
return;
}
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
if (ip_rt->ip_matcher != NULL) {
ip_matcher_free(ip_rt->ip_matcher);
ip_rt->ip_matcher = NULL;
}
if (ip_rt->htable != NULL) {
rcu_hash_free(ip_rt->htable);
ip_rt->htable = NULL;
}
if (ip_rt->item_htable != NULL) {
rcu_hash_free(ip_rt->item_htable);
ip_rt->item_htable = NULL;
}
if (ip_rt->hit_cnt != NULL) {
alignment_int64_array_free(ip_rt->hit_cnt);
ip_rt->hit_cnt = NULL;
}
if (ip_rt->scan_cnt != NULL) {
alignment_int64_array_free(ip_rt->scan_cnt);
ip_rt->scan_cnt = NULL;
}
FREE(ip_rt);
}
void ip_item_to_ip_rule(struct ip_item *item, struct ip_rule *rule)
{
if (IPv4 == item->addr_type) {
@@ -327,30 +323,44 @@ void ip_item_to_ip_rule(struct ip_item *item, struct ip_rule *rule)
rule->rule_id = item->item_id;
}
struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt)
{
return ip_rt->ex_data_rt;
}
int ip_runtime_update_row(struct ip_runtime *rt, char *key, size_t key_len,
int ip_runtime_update_row(struct ip_runtime *ip_rt, char *key, size_t key_len,
struct ip_item *item, int is_valid)
{
int ret = -1;
struct ex_data_runtime *ex_data_rt = rt->ex_data_rt;
void *data = NULL;
if (0 == is_valid) {
// delete
ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len);
if (ret < 0) {
data = rcu_hash_find(ip_rt->htable, key, key_len);
if (NULL == data) {
data = rcu_hash_updating_find(ip_rt->htable, key, key_len);
if (NULL == data) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] the key of ip rule(rule_id:%lld) not exist, can't be deleted from ip runtime htable",
__FUNCTION__, __LINE__, item->item_id);
return -1;
}
}
rcu_hash_del(ip_rt->htable, key, key_len);
} else {
// add
struct ex_data_container *ex_container = ex_data_container_new(NULL, (void *)item);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
data = rcu_hash_find(ip_rt->htable, key, key_len);
if (data != NULL) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] the key of ip rule(rule_id:%lld) already exist, can't be added to ip runtime htable",
__FUNCTION__, __LINE__, item->item_id);
return -1;
}
data = rcu_hash_updating_find(ip_rt->htable, key, key_len);
if (data != NULL) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] the key of ip rule(rule_id:%lld) already exist, can't be added to ip runtime htable",
__FUNCTION__, __LINE__, item->item_id);
return -1;
}
rcu_hash_add(ip_rt->htable, key, key_len, (void *)item);
}
return 0;
@@ -374,31 +384,40 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
}
int is_valid = get_column_value(line, valid_column);
//printf("<ip_runtime_update> item_id:%lld is_valid:%d\n", item_id, is_valid);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND(hh, ip_rt->item_hash, &item_id, sizeof(long long), item);
item = (struct maat_item *)rcu_hash_find(ip_rt->item_htable,
(char *)&item_id,
sizeof(item_id));
if (NULL == item) {
item = (struct maat_item *)rcu_hash_updating_find(ip_rt->item_htable,
(char *)&item_id,
sizeof(item_id));
if (NULL == item) {
return -1;
}
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
rcu_hash_del(ip_rt->item_htable, (char *)&item_id, sizeof(item_id));
} else {
//add
item = (struct maat_item *)rcu_hash_find(ip_rt->item_htable,
(char *)&item_id, sizeof(item_id));;
if (item) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] ip runtime add item(item_id:%lld) to item_htable failed, already exist",
__FUNCTION__, __LINE__, item_id);
return -1;
}
HASH_DELETE(hh, ip_rt->item_hash, item);
maat_garbage_bagging(ip_rt->ref_garbage_bin, u_para,
(void (*)(void *))maat_item_inner_free);
} else {
//add
HASH_FIND(hh, ip_rt->item_hash, &item_id, sizeof(long long), item);
item = (struct maat_item *)rcu_hash_updating_find(ip_rt->item_htable,
(char *)&item_id, sizeof(item_id));;
if (item) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] ip runtime add item %llu to item_hash failed, already exist",
"[%s:%d] ip runtime add item(item_id:%lld) to item_htable failed, already exist",
__FUNCTION__, __LINE__, item_id);
return -1;
}
@@ -409,8 +428,8 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
}
u_para = maat_item_inner_new(ip_item->group_id, item_id, 0);
item = maat_item_new(item_id, ip_item->group_id, u_para);
HASH_ADD(hh, ip_rt->item_hash, item_id, sizeof(long long), item);
item = maat_item_new(item_id, ip_item->group_id, u_para, maat_item_inner_free);
rcu_hash_add(ip_rt->item_htable, (char *)&(item_id), sizeof(item_id), item);
}
char *key = (char *)&item_id;
@@ -421,12 +440,6 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
ip_item = NULL;
}
return -1;
} else {
if (0 == is_valid) {
ip_rt->rule_num--;
} else {
ip_rt->rule_num++;
}
}
return 0;
@@ -438,31 +451,36 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name)
return -1;
}
int ret = 0;
struct ex_data_container **ex_container = NULL;
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
struct ex_data_runtime *ex_data_rt = ip_rt->ex_data_rt;
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (0 == rule_cnt) {
FREE(ex_container);
int updating_flag = rcu_hash_is_updating(ip_rt->htable);
if (0 == updating_flag) {
return 0;
}
struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt);
rcu_hash_commit(ip_rt->htable);
struct ip_rule *rules = NULL;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list(ip_rt->htable, &ex_data_array);
if (rule_cnt > 0) {
rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
struct ip_item *item = (struct ip_item *)ex_container[i]->custom_data;
struct ip_item *item = (struct ip_item *)ex_data_array[i];
//printf("<ip_runtime_commit> item_id:%lld\n", item->item_id);
ip_item_to_ip_rule(item, &rules[i]);
}
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
size_t mem_used = 0;
}
log_info(ip_rt->logger, MODULE_IP,
"table[%s] committing %zu ip rules for rebuilding ip_matcher engine",
table_name, rule_cnt);
int ret = 0;
size_t mem_used = 0;
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used, ip_rt->logger);
if (NULL == new_ip_matcher) {
log_error(ip_rt->logger, MODULE_IP,
@@ -473,12 +491,20 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name)
old_ip_matcher = ip_rt->ip_matcher;
ip_rt->ip_matcher = new_ip_matcher;
if (old_ip_matcher != NULL) {
maat_garbage_bagging(ip_rt->ref_garbage_bin, old_ip_matcher,
(void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ex_data_rt);
}
rcu_hash_commit(ip_rt->item_htable);
ip_rt->rule_num = rule_cnt;
if (rules != NULL) {
FREE(rules);
FREE(ex_container);
}
if (ex_data_array != NULL) {
FREE(ex_data_array);
}
return ret;
}
@@ -525,7 +551,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
}
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item,
int ret = maat_compile_state_update(ip_rt->item_htable, vtable_id, hit_item_ids, n_hit_item,
&group_hit_cnt, state);
if (ret < 0) {
return -1;

48
src/maat_ip_plugin.c
View File

@@ -128,7 +128,7 @@ void ip_plugin_schema_free(void *ip_plugin_schema)
schema->ex_schema = NULL;
}
FREE(schema);
free(schema);
}
struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema)
@@ -326,7 +326,7 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt,
} else {
// add
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)ip_plugin_rule);
struct ex_container *ex_container = ex_container_new(ex_data, (void *)ip_plugin_rule);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
return -1;
@@ -347,8 +347,8 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num,
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1);
ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id,
ex_data_container_free, logger);
ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_container_free,
garbage_bin, logger);
ip_plugin_rt->ref_garbage_bin = garbage_bin;
ip_plugin_rt->logger = logger;
@@ -435,32 +435,40 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name)
return -1;
}
int ret = 0;
struct ex_data_container **ex_container = NULL;
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt;
if (NULL == ex_data_rt) {
return -1;
}
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (0 == rule_cnt) {
FREE(ex_container);
int updating_flag = ex_data_runtime_is_updating(ex_data_rt);
if (0 == updating_flag) {
return 0;
}
struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt);
ex_data_runtime_commit(ex_data_rt);
struct ip_rule *rules = NULL;
struct ex_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
if (rule_cnt > 0) {
rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct ip_rule *)ex_container[i]->custom_data;
assert(rules[i].user_tag == ex_container[i] || rules[i].user_tag == NULL);
rules[i].user_tag = ex_container[i];
}
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
size_t mem_used = 0;
}
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"table[%s] committing %zu ip_plugin rules for rebuilding ip_matcher engine",
table_name, rule_cnt);
int ret = 0;
size_t mem_used = 0;
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used, ip_plugin_rt->logger);
if (NULL == new_ip_matcher) {
log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
@@ -471,12 +479,20 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name)
old_ip_matcher = ip_plugin_rt->ip_matcher;
ip_plugin_rt->ip_matcher = new_ip_matcher;
if (old_ip_matcher != NULL) {
maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher,
(void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ex_data_rt);
}
ip_plugin_rt->rule_num = rule_cnt;
if (rules != NULL) {
FREE(rules);
}
if (ex_container != NULL) {
FREE(ex_container);
}
return ret;
}
@@ -518,7 +534,7 @@ int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr
int n_result = ip_matcher_match(ip_plugin_rt->ip_matcher, &ip_data, results, n_ex_data);
for (int i = 0; i < n_result; i++) {
ex_data_array[i] = ex_data_runtime_get_ex_data_by_container(ip_plugin_rt->ex_data_rt,
(struct ex_data_container *)results[i].tag);
(struct ex_container *)results[i].tag);
}
return n_result;
}

40
src/maat_kv.c
View File

@@ -19,7 +19,7 @@ struct maat_kv_pair
{
char* key; //must be lower case.
size_t keylen;
int val;
long long val;
UT_hash_handle hh;
};
@@ -35,7 +35,7 @@ void strlowercase(const char* src, size_t src_len, char* dst, size_t dst_sz)
}
}
struct maat_kv_pair* maat_kv_pair_new(const char* key, size_t keylen, int value)
struct maat_kv_pair* maat_kv_pair_new(const char* key, size_t keylen, long long value)
{
struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1);
@@ -58,7 +58,7 @@ void maat_kv_pair_free(struct maat_kv_pair* kv)
struct maat_kv_store* maat_kv_store_new(void)
{
struct maat_kv_store* store = ALLOC(struct maat_kv_store, 1);
struct maat_kv_store *store = ALLOC(struct maat_kv_store, 1);
return store;
}
@@ -79,7 +79,7 @@ void maat_kv_store_free(struct maat_kv_store* store)
FREE(store);
}
int maat_kv_register_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int value)
int maat_kv_register_unNull(struct maat_kv_store* store, const char *key, size_t keylen, long long value)
{
if (keylen > MAAT_KV_MAX_KEY_LEN) {
return -1;
@@ -99,7 +99,7 @@ int maat_kv_register_unNull(struct maat_kv_store* store, const char* key, size_t
return 1;
}
int maat_kv_register(struct maat_kv_store* store, const char* key, int value)
int maat_kv_register(struct maat_kv_store* store, const char *key, long long value)
{
int ret = 0;
ret = maat_kv_register_unNull(store, key, strlen(key), value);
@@ -107,7 +107,7 @@ int maat_kv_register(struct maat_kv_store* store, const char* key, int value)
return ret;
}
int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int* value)
int maat_kv_read_unNull(struct maat_kv_store *store, const char* key, size_t keylen, long long *value)
{
struct maat_kv_pair *kv = NULL;
char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0};
@@ -126,11 +126,37 @@ int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t key
}
}
int maat_kv_read(struct maat_kv_store * store, const char * key, int * value)
int maat_kv_read(struct maat_kv_store *store, const char * key, long long *value)
{
return maat_kv_read_unNull(store, key, strlen(key), value);
}
int maat_kv_write_unNull(struct maat_kv_store* store, const char* key, size_t keylen, long long value)
{
struct maat_kv_pair *kv = NULL;
char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0};
if (keylen > MAAT_KV_MAX_KEY_LEN) {
return -1;
}
strlowercase(key, keylen, key_lowercase, sizeof(key_lowercase));
HASH_FIND(hh, store->hash, key_lowercase, keylen, kv);
if (kv) {
kv->val = value;
} else {
kv = maat_kv_pair_new(key, keylen, value);
HASH_ADD_KEYPTR(hh, store->hash, kv->key, keylen, kv);
}
return 1;
}
int maat_kv_write(struct maat_kv_store *store, const char *key, long long value)
{
return maat_kv_write_unNull(store, key, strlen(key), value);
}
struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *origin_map)
{
struct maat_kv_store *target = maat_kv_store_new();

21
src/maat_plugin.c
View File

@@ -142,7 +142,7 @@ void plugin_schema_free(void *plugin_schema)
schema->ex_schema = NULL;
}
FREE(schema);
free(schema);
}
int plugin_table_add_callback(void *plugin_schema, int table_id,
@@ -240,7 +240,8 @@ void *plugin_runtime_new(void *plugin_schema, int max_thread_num,
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
struct plugin_runtime *plugin_rt = ALLOC(struct plugin_runtime, 1);
plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id,
ex_data_container_free, logger);
ex_container_free,
garbage_bin, logger);
plugin_rt->ref_garbage_bin = garbage_bin;
plugin_rt->logger = logger;
@@ -281,7 +282,7 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
} else {
// add
void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL);
struct ex_container *ex_container = ex_container_new(ex_data, NULL);
ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
return -1;
@@ -388,16 +389,18 @@ int plugin_runtime_commit(void *plugin_runtime, const char *table_name)
}
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
struct ex_data_runtime *ex_data_rt = plugin_rt->ex_data_rt;
if (NULL == ex_data_rt) {
return -1;
}
struct ex_data_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(plugin_rt->ex_data_rt,
&ex_container);
if (rule_cnt == 0) {
FREE(ex_container);
int updating_flag = ex_data_runtime_is_updating(ex_data_rt);
if (0 == updating_flag) {
return 0;
}
ex_data_runtime_commit(plugin_rt->ex_data_rt);
ex_data_runtime_commit(ex_data_rt);
plugin_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt);
return 0;
}

36
src/maat_redis_monitor.c
View File

@@ -65,7 +65,7 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns,
&foreign_key_size);
if (NULL == p_foreign) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Get %s,%lu foreign keys failed: No %dth column",
"[%s:%d] Get %s,%lld foreign keys failed: No %dth column",
__FUNCTION__, __LINE__, p_rule->table_name, p_rule->rule_id,
foreign_columns[i]);
continue;
@@ -78,7 +78,7 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns,
if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Get %s,%lu foreign key failed: Invalid source prefix %s",
"[%s:%d] Get %s,%lld foreign key failed: Invalid source prefix %s",
__FUNCTION__, __LINE__, p_rule->table_name, p_rule->rule_id, p_foreign);
continue;
}
@@ -89,7 +89,7 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns,
if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) {
log_info(logger, MODULE_REDIS_MONITOR,
"[%s:%d] %s, %lu foreign key prefix %s is not recommended",
"[%s:%d] %s, %lld foreign key prefix %s is not recommended",
__FUNCTION__, __LINE__, p_rule->table_name, p_rule->rule_id, p_foreign);
}
@@ -201,7 +201,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Redis GET %s:%s,%lu failed, redis server error",
"[%s:%d] Redis GET %s:%s,%lld failed, redis server error",
__FUNCTION__, __LINE__, mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id);
error_happened = 1;
@@ -216,7 +216,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
failed_cnt++;
} else {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Redis GET %s:%s,%lu failed",
"[%s:%d] Redis GET %s:%s,%lld failed",
__FUNCTION__, __LINE__, mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id);
error_happened = 1;
@@ -774,7 +774,7 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Get %s,%lu foreign key %s content failed, redis server error",
"[%s:%d] Get %s,%lld foreign key %s content failed, redis server error",
__FUNCTION__, __LINE__,
rule_list[track[i].rule_idx].table_name,
rule_list[track[i].rule_idx].rule_id,
@@ -784,7 +784,7 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
if (reply->type != REDIS_REPLY_STRING) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Get %s,%lu foreign key %s content failed",
"[%s:%d] Get %s,%lld foreign key %s content failed",
__FUNCTION__, __LINE__,
rule_list[track[i].rule_idx].table_name,
rule_list[track[i].rule_idx].rule_id,
@@ -993,7 +993,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list,
for (i = 0; i < rule_num; i++) {
switch (s_rule[i].op) {
case MAAT_OP_ADD:
redisAppendCommand(c, "SET %s:%s,%lu %s",
redisAppendCommand(c, "SET %s:%s,%lld %s",
mr_key_prefix[MAAT_OP_ADD],
s_rule[i].table_name,
s_rule[i].rule_id,
@@ -1002,7 +1002,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list,
(*cnt)++;
append_cmd_cnt++;
//Allowing add duplicated members for rule id recycling.
redisAppendCommand(c, "RPUSH %s ADD,%s,%lu",
redisAppendCommand(c, "RPUSH %s ADD,%s,%lld",
transaction_list,
s_rule[i].table_name,
s_rule[i].rule_id);
@@ -1010,7 +1010,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list,
(*cnt)++;
append_cmd_cnt++;
if (s_rule[i].timeout > 0) {
redisAppendCommand(c, "ZADD %s %lld %s,%lu",
redisAppendCommand(c, "ZADD %s %lld %s,%lld",
mr_expire_sset,
s_rule[i].timeout,
s_rule[i].table_name,
@@ -1022,7 +1022,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list,
}
break;
case MAAT_OP_DEL:
redisAppendCommand(c, "RENAME %s:%s,%lu %s:%s,%lu",
redisAppendCommand(c, "RENAME %s:%s,%lld %s:%s,%lld",
mr_key_prefix[MAAT_OP_ADD],
s_rule[i].table_name,
s_rule[i].rule_id,
@@ -1033,7 +1033,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list,
(*cnt)++;
append_cmd_cnt++;
redisAppendCommand(c, "EXPIRE %s:%s,%lu %d",
redisAppendCommand(c, "EXPIRE %s:%s,%lld %d",
mr_key_prefix[MAAT_OP_DEL],
s_rule[i].table_name,
s_rule[i].rule_id,
@@ -1043,7 +1043,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list,
append_cmd_cnt++;
//NX: Don't update already exisiting elements. Always add new elements.
redisAppendCommand(c, "RPUSH %s DEL,%s,%lu",
redisAppendCommand(c, "RPUSH %s DEL,%s,%lld",
transaction_list,
s_rule[i].table_name,
s_rule[i].rule_id);
@@ -1052,7 +1052,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list,
append_cmd_cnt++;
// Try to remove from expiration sorted set, no matter wheather it exists or not.
redisAppendCommand(c, "ZREM %s %s,%lu",
redisAppendCommand(c, "ZREM %s %s,%lld",
mr_expire_sset,
s_rule[i].table_name,
s_rule[i].rule_id);
@@ -1061,7 +1061,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list,
append_cmd_cnt++;
// Try to remove from label sorted set, no matter wheather it exists or not.
redisAppendCommand(c, "ZREM %s %s,%lu",
redisAppendCommand(c, "ZREM %s %s,%lld",
mr_label_sset,
s_rule[i].table_name,
s_rule[i].rule_id);
@@ -1074,7 +1074,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list,
continue;
}
//s_rule[i].timeout>0 was checked by caller.
redisAppendCommand(c, "ZADD %s %lld %s,%lu",
redisAppendCommand(c, "ZADD %s %lld %s,%lld",
mr_expire_sset,
s_rule[i].timeout,
s_rule[i].table_name,
@@ -1194,7 +1194,7 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
rule_seq = expected_reply[i].s_rule_seq;
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] %s %s %lu failed, rule id maybe conflict or not exist",
"[%s:%d] %s %s %lld failed, rule id maybe conflict or not exist",
__FUNCTION__, __LINE__, mr_op_str[s_rule[rule_seq].op],
s_rule[rule_seq].table_name, s_rule[rule_seq].rule_id);
success_cnt--;
@@ -1219,7 +1219,7 @@ error_out:
for (i = 0; i < (unsigned int)serial_rule_num; i++) {
if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] %s %s %lu is not allowed due to lock contention",
"[%s:%d] %s %s %lld is not allowed due to lock contention",
__FUNCTION__, __LINE__, mr_op_str[MAAT_OP_RENEW_TIMEOUT],
s_rule[i].table_name, s_rule[i].rule_id);
}

66
src/maat_rule.c
View File

@@ -31,21 +31,24 @@
#define MODULE_MAAT_RULE module_name_str("maat.rule")
struct maat_item *maat_item_new(long long item_id, long long group_id, void *user_data)
struct maat_item *maat_item_new(long long item_id, long long group_id, void *user_data,
void (*user_data_free)(void *))
{
struct maat_item *item = NULL;
item = ALLOC(struct maat_item, 1);
item->group_id = group_id;
item->item_id = item_id;
item->user_data = user_data;
item->user_data_free = user_data_free;
return item;
}
void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *))
void maat_item_free(void *user_ctx, void *maat_item)
{
if (item_user_data_free && item->user_data) {
item_user_data_free(item->user_data);
struct maat_item *item = (struct maat_item *)maat_item;
if (item->user_data_free && item->user_data) {
item->user_data_free(item->user_data);
item->user_data = NULL;
}
@@ -206,7 +209,7 @@ error:
return ret;
}
struct maat_item_inner *maat_item_inner_new(long long group_id, long long item_id, int district_id)
struct maat_item_inner *maat_item_inner_new(long long group_id, long long item_id, long long district_id)
{
struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1);
item->magic_num = ITEM_RULE_MAGIC;
@@ -245,7 +248,7 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in
maat_rt->ref_tbl_mgr = maat_instance->tbl_mgr;
maat_rt->max_table_num = table_manager_table_count(maat_instance->tbl_mgr);
maat_rt->sequence_map = maat_kv_store_new();
maat_rt->logger = maat_instance->logger;
maat_rt->ref_garbage_bin = maat_instance->garbage_bin;
maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->nr_worker_thread);
@@ -262,6 +265,27 @@ void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger
maat_rt->last_update_time = time(NULL);
}
long long maat_runtime_get_sequence(struct maat_runtime *maat_rt, const char *key)
{
if (NULL == maat_rt || NULL == key) {
return -1;
}
long long sequence = 0;
int map_ret = maat_kv_read(maat_rt->sequence_map, key, &sequence);
if (map_ret < 0) {
maat_kv_register(maat_rt->sequence_map, key, sequence);
} else {
sequence++;
int ret = maat_kv_write(maat_rt->sequence_map, key, sequence);
if (ret < 0) {
return -1;
}
}
return sequence;
}
void maat_runtime_destroy(struct maat_runtime *maat_rt)
{
if (NULL == maat_rt) {
@@ -273,12 +297,23 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt)
maat_rt->ref_tbl_mgr = NULL;
}
if (maat_rt->sequence_map != NULL) {
maat_kv_store_free(maat_rt->sequence_map);
maat_rt->sequence_map = NULL;
}
if (maat_rt->ref_cnt != NULL) {
alignment_int64_array_free(maat_rt->ref_cnt);
maat_rt->ref_cnt = NULL;
}
FREE(maat_rt);
}
void maat_start_cb(long long new_version, int update_type, void *u_param)
{
struct maat *maat_instance = (struct maat *)u_param;
maat_instance->new_version = new_version;
if (update_type == MAAT_UPDATE_TYPE_FULL) {
maat_instance->creating_maat_rt = maat_runtime_create(new_version, maat_instance);
@@ -286,9 +321,22 @@ void maat_start_cb(long long new_version, int update_type, void *u_param)
maat_instance->maat_version = new_version;
}
struct maat_runtime *maat_rt = NULL;
if (maat_instance->creating_maat_rt != NULL) {
maat_rt = maat_instance->creating_maat_rt;
} else {
maat_rt = maat_instance->maat_rt;
}
size_t table_cnt = table_manager_table_count(maat_instance->tbl_mgr);
for (size_t i = 0; i < table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, i);
if (table_type == TABLE_TYPE_COMPILE) {
//compile runtime need a reference to maat runtime
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, i);
compile_runtime_init(compile_rt, maat_rt);
}
if (table_type != TABLE_TYPE_PLUGIN) {
continue;
}
@@ -386,7 +434,12 @@ void maat_finish_cb(void *u_param)
"Inc config version %llu load %d entries complete",
maat_instance->maat_rt->version,
maat_instance->maat_rt->rule_num);
} else {
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Version %d has no valid rules, plugin callback complete.", maat_instance->maat_version);
}
maat_instance->new_version = INVALID_VERSION;
}
void *rule_monitor_loop(void *arg)
@@ -524,6 +577,7 @@ void *rule_monitor_loop(void *arg)
}
}
log_handle_destroy(maat_instance->logger);
FREE(maat_instance);
return NULL;

42
src/maat_table.c
View File

@@ -372,7 +372,7 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo
goto error;
}
ret = maat_kv_read(reserved_word_map, item->valuestring, (int*)&(ptable->table_type));
ret = maat_kv_read(reserved_word_map, item->valuestring, (long long *)&(ptable->table_type));
if (ret < 0) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table:%s table_type %s is illegal",
@@ -462,22 +462,17 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
log_error(logger, MODULE_TABLE,
"[%s:%d] invalid json content in %s",
__FUNCTION__, __LINE__, table_info_path);
free(json_buff);
FREE(json_buff);
cJSON_Delete(root);
return NULL;
}
struct maat_kv_store *reserved_word_map = maat_kv_store_new();
register_reserved_word(reserved_word_map);
struct table_manager *tbl_mgr = ALLOC(struct table_manager, 1);
tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger);
tbl_mgr->logger = logger;
tbl_mgr->tablename2id_map = maat_kv_store_new();
tbl_mgr->ref_garbage_bin = garbage_bin;
int default_compile_table_id = MAX_TABLE_NUM;
int g2g_table_id = MAX_TABLE_NUM;
for (int i = 0; i < json_array_size; i++) {
json = cJSON_GetArrayItem(root, i);
@@ -485,13 +480,25 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
ret = register_tablename2id(json, tbl_mgr->tablename2id_map, logger);
if (ret < 0) {
log_error(logger, MODULE_TABLE,
"[%s:%d] register_tablename2id failed",
__FUNCTION__, __LINE__);
"[%s:%d] register_tablename2id failed", __FUNCTION__, __LINE__);
FREE(json_buff);
cJSON_Delete(root);
maat_kv_store_free(tbl_mgr->tablename2id_map);
for (int idx = 0; idx < tbl_mgr->n_accept_tag; idx++) {
FREE(tbl_mgr->accept_tags[idx].tag_name);
FREE(tbl_mgr->accept_tags[idx].tag_val);
}
FREE(tbl_mgr);
return NULL;
}
}
}
int default_compile_table_id = MAX_TABLE_NUM;
int g2g_table_id = MAX_TABLE_NUM;
struct maat_kv_store *reserved_word_map = maat_kv_store_new();
register_reserved_word(reserved_word_map);
for (int i = 0; i < json_array_size; i++) {
json = cJSON_GetArrayItem(root, i);
@@ -534,9 +541,9 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
log_info(logger, MODULE_TABLE, "default compile table id: %d", default_compile_table_id);
log_info(logger, MODULE_TABLE, "group2group table id: %d", g2g_table_id);
FREE(json_buff);
maat_kv_store_free(reserved_word_map);
cJSON_Delete(root);
FREE(json_buff);
return tbl_mgr;
}
@@ -645,6 +652,9 @@ void table_manager_destroy(struct table_manager *tbl_mgr)
enum table_type table_type = table_manager_get_table_type(tbl_mgr, i);
maat_table_schema_free(schema, table_type);
tbl_mgr->tbl[i]->schema = NULL;
maat_table_free(tbl_mgr->tbl[i]);
tbl_mgr->tbl[i] = NULL;
}
for (size_t i = 0; i < tbl_mgr->n_accept_tag; i++) {
@@ -668,7 +678,7 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name)
return -1;
}
int table_id = -1;
long long table_id = -1;
int ret = maat_kv_read(tbl_mgr->tablename2id_map, name, &table_id);
if (ret < 0) {
log_error(tbl_mgr->logger, MODULE_TABLE,
@@ -677,7 +687,7 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name)
return -1;
}
return table_id;
return (int)table_id;
}
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id)
@@ -746,14 +756,14 @@ int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *t
}
int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district,
size_t district_len, int *district_id)
size_t district_len, long long *district_id)
{
return maat_kv_read_unNull(tbl_mgr->district_map, district, district_len, district_id);
}
int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district)
long long table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district)
{
int district_id = DISTRICT_ANY;
long long district_id = DISTRICT_ANY;
int map_ret = maat_kv_read(tbl_mgr->district_map, district, &district_id);
if (map_ret < 0) {

76
src/rcu_hash.c
View File

@@ -12,6 +12,7 @@
#include <pthread.h>
#include <unistd.h>
#include <sys/queue.h>
#include <stdio.h>
#include "rcu_hash.h"
#include "maat_utils.h"
@@ -52,7 +53,7 @@ struct rcu_hash_node {
UT_hash_handle hh_b;
};
void rcu_hash_garbage_queue_free(struct rcu_hash_garbage_q* garbage_q)
void rcu_hash_garbage_queue_free(struct rcu_hash_garbage_q *garbage_q)
{
struct rcu_hash_garbage_bag *p = NULL;
@@ -179,28 +180,29 @@ void rcu_hash_add(struct rcu_hash_table *htable, const char *key,
return;
}
struct rcu_hash_node *tmp = NULL;
struct rcu_hash_node *node = ALLOC(struct rcu_hash_node, 1);
if (!htable->is_updating) {
rcu_hash_commit_prepare(htable);
}
node->key = (char *)malloc(sizeof(char) * key_len);
struct rcu_hash_node *tmp = NULL;
if (htable->effective_hash == 'a') {
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, tmp);
} else {
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, tmp);
}
if (NULL == tmp) {
struct rcu_hash_node *node = ALLOC(struct rcu_hash_node, 1);
node->key = ALLOC(char, key_len);
memcpy(node->key, key, key_len);
node->key_len = key_len;
node->data = data;
node->htable = htable;
if (!htable->is_updating) {
rcu_hash_commit_prepare(htable);
}
if (htable->effective_hash == 'a') {
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, tmp);
if (NULL == tmp) {
HASH_ADD_KEYPTR(hh_b, htable->hashmap_b, key, key_len, node);
}
HASH_ADD_KEYPTR(hh_b, htable->hashmap_b, node->key, node->key_len, node);
} else {
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, tmp);
if (NULL == tmp) {
HASH_ADD_KEYPTR(hh_a, htable->hashmap_a, key, key_len, node);
HASH_ADD_KEYPTR(hh_a, htable->hashmap_a, node->key, node->key_len, node);
}
}
}
@@ -257,6 +259,28 @@ void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_l
return NULL;
}
void *rcu_hash_updating_find(struct rcu_hash_table *htable, const char *key, size_t key_len)
{
if (NULL == htable || NULL == key || 0 == key_len) {
return NULL;
}
struct rcu_hash_node *node = NULL;
if (htable->effective_hash == 'a') {
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, node);
if (node != NULL) {
return node->data;
}
} else {
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, node);
if (node != NULL) {
return node->data;
}
}
return NULL;
}
size_t rcu_hash_count(struct rcu_hash_table *htable)
{
if (NULL == htable) {
@@ -270,17 +294,13 @@ size_t rcu_hash_count(struct rcu_hash_table *htable)
}
}
size_t rcu_hash_updating_count(struct rcu_hash_table *htable)
int rcu_hash_is_updating(struct rcu_hash_table *htable)
{
if (NULL == htable) {
return 0;
}
if (htable->effective_hash == 'a') {
return HASH_CNT(hh_b, htable->hashmap_b);
} else {
return HASH_CNT(hh_a, htable->hashmap_a);
}
return htable->is_updating;
}
void rcu_hash_commit(struct rcu_hash_table *htable)
@@ -289,10 +309,6 @@ void rcu_hash_commit(struct rcu_hash_table *htable)
return;
}
if (!htable->is_updating) {
return;
}
struct rcu_hash_node *node = NULL;
struct rcu_hash_node *tmp = NULL;
@@ -324,23 +340,23 @@ void rcu_hash_commit(struct rcu_hash_table *htable)
pthread_mutex_unlock(&htable->update_mutex);
}
size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array)
size_t rcu_hash_list(struct rcu_hash_table *htable, void ***data_array)
{
size_t i = 0;
size_t node_cnt = 0;
struct rcu_hash_node *node = NULL, *tmp = NULL;
if (htable->effective_hash == 'a') {
node_cnt = HASH_CNT(hh_b, htable->hashmap_b);
node_cnt = HASH_CNT(hh_a, htable->hashmap_a);
*data_array = ALLOC(void *, node_cnt);
HASH_ITER(hh_b, htable->hashmap_b, node, tmp) {
HASH_ITER(hh_a, htable->hashmap_a, node, tmp) {
(*data_array)[i] = node->data;
i++;
}
} else {
node_cnt = HASH_CNT(hh_a, htable->hashmap_a);
node_cnt = HASH_CNT(hh_b, htable->hashmap_b);
*data_array = ALLOC(void *, node_cnt);
HASH_ITER(hh_a, htable->hashmap_a, node, tmp) {
HASH_ITER(hh_b, htable->hashmap_b, node, tmp) {
(*data_array)[i] = node->data;
i++;
}

28
test/adapter_hs_gtest.cpp
View File

@@ -79,7 +79,7 @@ static size_t hex2bin(char *hex, int hex_len, char *binary, size_t size)
return resultlen;
}
int parse_config_file(const char *filename, and_expr_t exprs[], size_t *n_expr)
int parse_config_file(const char *filename, struct hs_expr exprs[], size_t *n_expr)
{
unsigned char *json_buff = NULL;
size_t json_buff_size = 0;
@@ -155,6 +155,7 @@ int parse_config_file(const char *filename, and_expr_t exprs[], size_t *n_expr)
pat_str, pat_str_len);
memcpy(exprs[i].patterns[j].pat, pat_str, pat_str_len);
free(pat_str);
exprs[i].patterns[j].pat_len = pat_str_len;
} else {
memcpy(exprs[i].patterns[j].pat, item->valuestring,
@@ -187,10 +188,11 @@ int parse_config_file(const char *filename, and_expr_t exprs[], size_t *n_expr)
*n_expr = rule_cnt;
next:
cJSON_Delete(root);
FREE(json_buff);
return ret;
}
void expr_array_free(and_expr_t expr_array[], size_t n_expr_array)
void expr_array_free(struct hs_expr expr_array[], size_t n_expr_array)
{
for (size_t i = 0; i < n_expr_array; i++) {
for (size_t j = 0; j < expr_array[i].n_patterns; j++) {
@@ -205,7 +207,7 @@ void expr_array_free(and_expr_t expr_array[], size_t n_expr_array)
TEST(block_mode_initialize, invalid_input_parameter)
{
struct adapter_hs *hs_instance = NULL;
and_expr_t exprs[64];
struct hs_expr exprs[64];
/* case1: invalid scan_mode parameter */
hs_instance = adapter_hs_initialize(HS_SCAN_MODE_MAX, HS_PATTERN_TYPE_REG,
@@ -225,7 +227,7 @@ TEST(block_mode_initialize, invalid_input_parameter)
TEST(block_mode_scan, invalid_input_parameter)
{
and_expr_t expr_array[64];
struct hs_expr expr_array[64];
size_t n_expr_array = 0;
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG,
@@ -254,7 +256,7 @@ TEST(block_mode_scan, invalid_input_parameter)
TEST(block_mode_scan, literal_sub_has_normal_offset)
{
and_expr_t expr_array[64] = {0};
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
@@ -295,7 +297,7 @@ TEST(block_mode_scan, literal_sub_has_normal_offset)
TEST(block_mode_scan, literal_sub_has_left_unlimit_offset)
{
and_expr_t expr_array[64] = {0};
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
@@ -336,7 +338,7 @@ TEST(block_mode_scan, literal_sub_has_left_unlimit_offset)
TEST(block_mode_scan, literal_sub_has_right_unlimit_offset)
{
and_expr_t expr_array[64] = {0};
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
@@ -392,7 +394,7 @@ TEST(block_mode_scan, literal_sub_has_right_unlimit_offset)
TEST(block_mode_scan, literal_sub_with_no_offset)
{
and_expr_t expr_array[64] = {0};
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
@@ -440,7 +442,7 @@ TEST(block_mode_scan, literal_sub_with_no_offset)
TEST(block_mode_scan, literal_exactly)
{
and_expr_t expr_array[64] = {0};
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
@@ -482,7 +484,7 @@ TEST(block_mode_scan, literal_exactly)
TEST(block_mode_scan, literal_prefix)
{
and_expr_t expr_array[64] = {0};
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
@@ -533,7 +535,7 @@ TEST(block_mode_scan, literal_prefix)
TEST(block_mode_scan, literal_suffix)
{
and_expr_t expr_array[64] = {0};
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
@@ -584,7 +586,7 @@ TEST(block_mode_scan, literal_suffix)
TEST(block_mode_scan, literal_sub_with_hexbin)
{
and_expr_t expr_array[64] = {0};
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
@@ -617,7 +619,7 @@ TEST(block_mode_scan, literal_sub_with_hexbin)
TEST(block_mode_scan, literal_with_chinese)
{
and_expr_t expr_array[64] = {0};
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);

5
test/maat_api_gtest.cpp
View File

@@ -1,10 +1,7 @@
#include "maat.h"
#include "maat_rule.h"
#include <gtest/gtest.h>
TEST(EQ_Test, Always_True) {
EXPECT_EQ(1, 1);
}
int main(int argc, char ** argv)
{

14
test/maat_ex_data_gtest.cpp
View File

@@ -4,6 +4,7 @@
#include "json2iris.h"
#include "maat_command.h"
#include "maat_ex_data.h"
#include "maat_garbage_collection.h"
#include <gtest/gtest.h>
#include <limits.h>
@@ -55,8 +56,8 @@ TEST(EXDataRuntime, Update) {
ASSERT_GT(table_id, 0);
int ex_data_counter = 0;
struct ex_data_runtime *ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free, g_logger);
struct maat_garbage_bin *garbage_bin = maat_garbage_bin_new(10);
struct ex_data_runtime *ex_data_rt = ex_data_runtime_new(table_id, ex_container_free, garbage_bin, g_logger);
struct ex_data_schema *ex_schema = ex_data_schema_new(ex_data_new_cb, ex_data_free_cb, ex_data_dup_cb,
0, &ex_data_counter);
ex_data_runtime_set_schema(ex_data_rt, ex_schema);
@@ -67,7 +68,7 @@ TEST(EXDataRuntime, Update) {
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row1, key1, key1_len);
EXPECT_EQ(ex_data_counter, 1);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL);
struct ex_container *ex_container = ex_container_new(ex_data, NULL);
int ret = ex_data_runtime_add_ex_container(ex_data_rt, key1, key1_len, ex_container);
EXPECT_EQ(ret, 0);
@@ -75,7 +76,7 @@ TEST(EXDataRuntime, Update) {
const char *key2 = "192.168.0.2";
size_t key2_len = strlen(key2);
ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row2, key2, key2_len);
ex_container = ex_data_container_new(ex_data, NULL);
ex_container = ex_container_new(ex_data, NULL);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key2, key2_len, ex_container);
EXPECT_EQ(ret, 0);
@@ -94,6 +95,10 @@ TEST(EXDataRuntime, Update) {
info = (struct user_info *)res_data2;
EXPECT_EQ(0, strcmp(info->name, "liyanhong"));
EXPECT_EQ(info->id, 2);
maat_garbage_bin_free(garbage_bin);
ex_data_runtime_free(ex_data_rt);
ex_data_schema_free(ex_schema);
}
int main(int argc, char ** argv)
@@ -119,6 +124,7 @@ int main(int argc, char ** argv)
ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path),
NULL, NULL, g_logger);
FREE(json_buff);
EXPECT_NE(ret, -1);
}

1697
test/maat_framework_gtest.cpp
View File

File diff suppressed because it is too large Load Diff

17
test/maat_input_mode_gtest.cpp
View File

@@ -25,6 +25,7 @@ TEST(json_mode, maat_scan_string) {
ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path),
NULL, NULL, g_logger);
FREE(json_buff);
EXPECT_NE(ret, -1);
}
@@ -48,11 +49,18 @@ TEST(json_mode, maat_scan_string) {
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 182);
long long clause_id = maat_runtime_get_sequence(maat_instance->maat_rt, "test_seq");
EXPECT_EQ(clause_id, 0);
clause_id = maat_runtime_get_sequence(maat_instance->maat_rt, "test_seq");
EXPECT_EQ(clause_id, 1);
clause_id = maat_runtime_get_sequence(maat_instance->maat_rt, "test_seq");
EXPECT_EQ(clause_id, 2);
maat_options_free(opts);
maat_state_free(&state);
maat_free(maat_instance);
}
#if 0
TEST(iris_mode, maat_scan_string) {
char tmp_iris_path[512] = {0};
char json_iris_path[512] = {0};
@@ -67,6 +75,7 @@ TEST(iris_mode, maat_scan_string) {
ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path),
NULL, NULL, g_logger);
FREE(json_buff);
EXPECT_NE(ret, -1);
}
@@ -140,7 +149,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name,
buff, absolute_expire_time);
line_idx++;
FREE(str1);
FREE(buff);
return 0;
}
@@ -169,6 +179,7 @@ TEST(redis_mode, maat_scan_string) {
ret = json2iris(json_buff, json_filename, c, tmp_iris_path, sizeof(tmp_iris_path),
NULL, NULL, g_logger);
FREE(json_buff);
EXPECT_NE(ret, -1);
}
@@ -219,7 +230,7 @@ TEST(redis_mode, maat_scan_string) {
maat_state_free(&state);
maat_free(maat_instance);
}
#endif
int main(int argc, char ** argv)
{
int ret=0;

146
test/rcu_hash_gtest.cpp
View File

@@ -3,6 +3,8 @@
#include <gtest/gtest.h>
struct rcu_hash_table *g_htable = NULL;
struct user_data {
int id;
char name[32];
@@ -44,10 +46,15 @@ TEST(rcu_hash_add_one_node, single_thread) {
EXPECT_EQ(ret, 0);
void **data_array = NULL;
ret = rcu_hash_list_updating_data(htable, &data_array);
rcu_hash_commit(htable);
ret = rcu_hash_list(htable, &data_array);
EXPECT_EQ(ret, 1);
rcu_hash_commit(htable);
if (data_array != NULL) {
free(data_array);
data_array = NULL;
}
/* find in hash after commit */
res = rcu_hash_find(htable, key, key_len);
@@ -63,9 +70,6 @@ TEST(rcu_hash_add_one_node, single_thread) {
ret = rcu_hash_garbage_queue_len(htable);
EXPECT_EQ(ret, 0);
ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 0);
rcu_hash_free(htable);
}
@@ -119,11 +123,22 @@ TEST(rcu_hash_add_multi_node, single_thread) {
EXPECT_EQ(ret, 0);
void **data_array = NULL;
ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 4);
ret = rcu_hash_list(htable, &data_array);
EXPECT_EQ(ret, 0);
if (data_array != NULL) {
free(data_array);
data_array = NULL;
}
rcu_hash_commit(htable);
ret = rcu_hash_list(htable, &data_array);
EXPECT_EQ(ret, 4);
if (data_array != NULL) {
free(data_array);
data_array = NULL;
}
/* find in hash after commit */
res = rcu_hash_find(htable, key0, key0_len);
EXPECT_TRUE(res != NULL);
@@ -155,9 +170,6 @@ TEST(rcu_hash_add_multi_node, single_thread) {
ret = rcu_hash_garbage_queue_len(htable);
EXPECT_EQ(ret, 0);
ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 0);
rcu_hash_free(htable);
}
@@ -177,8 +189,12 @@ TEST(rcu_hash_del_one_node, single_thread) {
rcu_hash_add(htable, key, key_len, (void *)data);
void **data_array = NULL;
int ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 1);
int ret = rcu_hash_list(htable, &data_array);
EXPECT_EQ(ret, 0);
if (data_array != NULL) {
free(data_array);
data_array = NULL;
}
/* find in hash before commit */
void *res = rcu_hash_find(htable, key, key_len);
@@ -192,8 +208,12 @@ TEST(rcu_hash_del_one_node, single_thread) {
rcu_hash_del(htable, key, key_len);
ret = rcu_hash_list_updating_data(htable, &data_array);
ret = rcu_hash_list(htable, &data_array);
EXPECT_EQ(ret, 0);
if (data_array != NULL) {
free(data_array);
data_array = NULL;
}
rcu_hash_commit(htable);
@@ -297,10 +317,110 @@ TEST(rcu_hash_del_multi_node, single_thread) {
rcu_hash_free(htable);
}
TEST(global_rcu_hash_add, single_thread) {
EXPECT_TRUE(g_htable != NULL);
struct user_data *data1 = ALLOC(struct user_data, 1);
data1->id = 101;
char name1[64] = "www.baidu.com";
memcpy(data1->name, name1, strlen(name1));
char key1[64] = "http_url";
size_t key1_len = strlen(key1);
rcu_hash_add(g_htable, key1, key1_len, (void *)data1);
int ret = rcu_hash_count(g_htable);
EXPECT_EQ(ret, 0);
rcu_hash_commit(g_htable);
ret = rcu_hash_count(g_htable);
EXPECT_EQ(ret, 1);
void *res = rcu_hash_find(g_htable, key1, key1_len);
EXPECT_TRUE(res != NULL);
}
TEST(global_rcu_hash_find, single_thread) {
EXPECT_TRUE(g_htable != NULL);
const char *key = "http_url";
size_t key_len = strlen(key);
void *res = rcu_hash_find(g_htable, key, key_len);
EXPECT_TRUE(res != NULL);
}
TEST(global_rcu_hash_del, single_thread) {
EXPECT_TRUE(g_htable != NULL);
const char *key = "http_url";
size_t key_len = strlen(key);
void *res = rcu_hash_find(g_htable, key, key_len);
EXPECT_TRUE(res != NULL);
rcu_hash_del(g_htable, key, key_len);
rcu_hash_commit(g_htable);
res = rcu_hash_find(g_htable, key, key_len);
EXPECT_TRUE(res == NULL);
}
TEST(rcu_hash_updating_find, single_thread) {
struct rcu_hash_table *htable = rcu_hash_new(data_free);
EXPECT_TRUE(htable != NULL);
//add data1
struct user_data *data1 = ALLOC(struct user_data, 1);
data1->id = 101;
char name1[64] = "www.baidu.com";
memcpy(data1->name, name1, strlen(name1));
char key[64] = "http_url";
size_t key_len = strlen(key);
rcu_hash_add(htable, key, key_len, (void *)data1);
void *res = rcu_hash_find(htable, key, key_len);
EXPECT_TRUE(res == NULL);
res = rcu_hash_updating_find(htable, key, key_len);
EXPECT_TRUE(res != NULL);
//del data1
rcu_hash_del(htable, key, key_len);
res = rcu_hash_updating_find(htable, key, key_len);
EXPECT_TRUE(res == NULL);
//add data2
struct user_data *data2 = ALLOC(struct user_data, 1);
data2->id = 102;
char name2[64] = "www.google.com";
memcpy(data2->name, name2, strlen(name2));
rcu_hash_add(htable, key, key_len, (void *)data2);
res = rcu_hash_updating_find(htable, key, key_len);
EXPECT_TRUE(res != NULL);
rcu_hash_commit(htable);
res = rcu_hash_find(htable, key, key_len);
struct user_data *tmp_data = (struct user_data *)res;
EXPECT_EQ(tmp_data->id, 102);
EXPECT_STREQ(tmp_data->name, "www.google.com");
rcu_hash_free(htable);
}
int main(int argc, char ** argv)
{
int ret=0;
::testing::InitGoogleTest(&argc, argv);
g_htable = rcu_hash_new(data_free);
ret=RUN_ALL_TESTS();
rcu_hash_free(g_htable);
return ret;
}

4
tools/maat_redis_tool.cpp
View File

@@ -231,7 +231,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time);
line_idx++;
FREE(str1);
FREE(buff);
return 0;
}
@@ -308,6 +309,7 @@ int main(int argc, char * argv[])
}
ret = json2iris(json_buff, json_file, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger);
FREE(json_buff);
if (ret < 0) {
printf("Invalid json format.\n");
}