gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unfinished work

Browse Source
This commit is contained in:
liuwentan
2023-02-03 17:28:14 +08:00
parent cca7d882e1
commit 57f0a0581a
45 changed files with 2338 additions and 1522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
include/maat/maat.h
View File

@@ -32,33 +32,22 @@ struct maat_hit_path {
int item_id;
int sub_group_id;
int top_group_id;
int virtual_table_id; // 0 is not a virtual table.
int vtable_id; // 0 is not a virtual table.
int compile_id;
};
struct maat_hit_object {
int virtual_table_id;
int vtable_id;
int group_id;
};
enum maat_scan_opt
{
MAAT_SET_SCAN_DISTRICT = 1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default.
MAAT_SET_SCAN_LAST_ITEM, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination.
MAAT_SET_SCAN_COMPILE_TABLE_ID, //Caller can specify which compile table to use
MAAT_GET_SCAN_HIT_PATH, //VALUE is struct maat_hit_path*, an array of struct maat_hit_path, SIZE= sizeof(struct maat_hit_path)*N,
//Maat_get_scan_status returns actual got number.
MAAT_GET_SCAN_HIT_OBJECTS //VALUE is struct maat_hit_object*, an array of struct maat_hit_object, SIZE= sizeof(struct maat_hit_object)*N,
enum maat_scan_status {
MAAT_SCAN_ERR = -1, //scan error
MAAT_SCAN_OK, //scan but not hit(group or compile)
MAAT_SCAN_HALF_HIT, //half hit: hit group, not hit compile
MAAT_SCAN_HIT //scan hit compile
};
#define MAAT_OK 0 //scan but not hit(group or compile)
#define MAAT_ERR -1 //scan error
#define MAAT_HALF_HIT 1 //half hit: hit group, not hit compile
#define MAAT_HIT 2 //scan hit compile
#define MAAT_RULE_UPDATE_TYPE_FULL 1
#define MAAT_RULE_UPDATE_TYPE_INC 2
typedef void maat_start_callback_t(int update_type, void *u_param);
typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para);
typedef void maat_finish_callback_t(void *u_para);
@@ -125,8 +114,14 @@ struct maat_state;
* @param n_hit_result: the number of hit compile id
* @param state: scan mid status
*
* @retval MAAT_ERR/MAAT_OK/MAAT_HALF_HIT
* @retval MAAT_SCAN_ERR
* MAAT_SCAN_OK
* MAAT_SCAN_HALF_HIT
* MAAT_SCAN_HIT
*/
int maat_scan_flag(struct maat *instance, int table_id, int thread_id,
uint64_t flag, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
@@ -152,15 +147,19 @@ int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* dat
void maat_scan_stream_close(struct maat_stream **stream);
/* maat state API */
int maat_state_set_scan_district(struct maat *instance, struct maat_state **state, const char *district, size_t district_len);
int maat_state_set_scan_district(struct maat *instance, struct maat_state **state,
const char *district, size_t district_len);
int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state);
int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, int compile_table_id);
int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state,
int compile_table_id);
int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state, struct maat_hit_path *paths, size_t n_path);
int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state,
struct maat_hit_path *paths, size_t n_path);
int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, struct maat_hit_object *objs, size_t n_obj);
int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state,
struct maat_hit_object *objs, size_t n_obj);
void maat_state_free(struct maat_state **state);

36
include/utils.h
View File

@@ -1,36 +0,0 @@
/*
**********************************************************************************************
* File: maat_utils.h
* Description: maat utils entry
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _UTILS_H_
#define _UTILS_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include <stdlib.h>
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
#define FREE(ptr) \
{ \
if (ptr) \
{ \
free(ptr); \
ptr = NULL; \
} \
}
#ifdef __cpluscplus
}
#endif
#endif

106
scanner/adapter_hs.cpp
View File

@@ -16,7 +16,6 @@
#include "adapter_hs.h"
#include "uthash/utarray.h"
#include "uthash/uthash.h"
#include "utils.h"
#include "maat_utils.h"
#include "bool_matcher.h"
@@ -43,7 +42,7 @@ struct adapter_hs_runtime {
/* adapter_hs instance */
struct adapter_hs {
size_t nr_worker_threads;
size_t n_worker_thread;
size_t n_expr;
size_t n_patterns;
struct adapter_hs_runtime *hs_rt;
@@ -59,25 +58,27 @@ struct adapter_hs_stream {
UT_array *pattern_id_set;
};
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_threads, int max_pattern_type,
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt,
size_t n_worker_thread, int pattern_type,
struct log_handle *logger)
{
hs_database_t *database = NULL;
hs_rt->scratchs = ALLOC(hs_scratch_t *, nr_worker_threads);
hs_rt->scratchs = ALLOC(hs_scratch_t *, n_worker_thread);
if (max_pattern_type == PATTERN_TYPE_STR) {
if (pattern_type == PATTERN_TYPE_STR) {
database = hs_rt->literal_db;
} else {
database = hs_rt->regex_db;
}
if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS, "ERROR: Unable to allocate scratch space. Exiting.");
log_error(logger, MODULE_ADAPTER_HS,
"ERROR: Unable to allocate scratch space. Exiting.");
hs_free_database(database);
return -1;
}
for (size_t i = 1; i < nr_worker_threads; i++) {
for (size_t i = 1; i < n_worker_thread; i++) {
hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]);
if (err != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS, "Unable to clone scratch prototype");
@@ -119,7 +120,8 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
scan_mode, NULL, &hs_rt->literal_db, &compile_err);
if (err != HS_SUCCESS) {
if (compile_err) {
log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message);
log_error(logger, MODULE_ADAPTER_HS,
"%s compile error: %s", __func__, compile_err->message);
}
hs_free_compile_error(compile_err);
@@ -133,7 +135,8 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
scan_mode, NULL, &hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) {
if (compile_err) {
log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message);
log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s",
__func__, compile_err->message);
}
hs_free_compile_error(compile_err);
goto error;
@@ -186,12 +189,14 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt
FREE(hs_cd);
}
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array,
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t n_worker_thread,
and_expr_t *expr_array, size_t n_expr_array,
struct log_handle *logger)
{
if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) ||
0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) {
log_error(logger, MODULE_ADAPTER_HS, "%s input parameters illegal!", __func__);
0 == n_worker_thread || NULL == expr_array || 0 == n_expr_array) {
log_error(logger, MODULE_ADAPTER_HS,
"%s input parameters illegal!", __func__);
return NULL;
}
@@ -202,7 +207,8 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
for (size_t i = 0; i < n_expr_array; i++) {
if (expr_array[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
log_error(logger, MODULE_ADAPTER_HS,
"the number of patterns in one expression should less than %d", MAX_EXPR_PATTERN_NUM);
"the number of patterns in one expression should less than %d",
MAX_EXPR_PATTERN_NUM);
return NULL;
}
@@ -212,7 +218,8 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
} else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) {
regex_pattern_num++;
} else {
log_error(logger, MODULE_ADAPTER_HS, "unknown pattern type: %d", expr_array[i].patterns[j].type);
log_error(logger, MODULE_ADAPTER_HS, "unknown pattern type: %d",
expr_array[i].patterns[j].type);
return NULL;
}
}
@@ -283,7 +290,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
size_t mem_size = 0;
struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1);
hs_instance->nr_worker_threads = nr_worker_threads;
hs_instance->n_worker_thread = n_worker_thread;
hs_instance->n_patterns = pattern_id;
hs_instance->n_expr = n_expr_array;
hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
@@ -321,7 +328,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
max_patterns_type = PATTERN_TYPE_REG;
}
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, nr_worker_threads, max_patterns_type, logger);
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, max_patterns_type, logger);
if (ret < 0) {
goto error;
}
@@ -349,7 +356,7 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance)
}
if (hs_instance->hs_rt->scratchs != NULL) {
for (size_t i = 0; i < hs_instance->nr_worker_threads; i++) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
if (hs_instance->hs_rt->scratchs[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->scratchs[i]);
}
@@ -399,10 +406,13 @@ int matched_event_cb(unsigned int id, unsigned long long from,
return 0;
}
int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *data, size_t data_len,
int results[], size_t *n_results)
int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
const char *data, size_t data_len,
struct hs_scan_result *results,
size_t *n_results)
{
if (NULL == hs_instance || NULL == data || (0 == data_len) || NULL == results || NULL == n_results) {
if (NULL == hs_instance || NULL == data || (0 == data_len) ||
NULL == results || NULL == n_results) {
return -1;
}
@@ -416,7 +426,8 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d
int err_count = 0;
if (hs_rt->literal_db != NULL) {
err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set);
err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch,
matched_event_cb, pattern_id_set);
if (err != HS_SUCCESS) {
//log_error()
err_count++;
@@ -424,7 +435,8 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d
}
if (hs_rt->regex_db != NULL) {
err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set);
err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch,
matched_event_cb, pattern_id_set);
if (err != HS_SUCCESS) {
//log_error()
err_count++;
@@ -442,18 +454,28 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d
items[i] = *(unsigned long long *)utarray_eltptr(pattern_id_set, i);
}
size_t matched_index = 0;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr);
size_t bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_instance->n_expr);
int ret = 0;
int matched_index = 0;
struct bool_expr_match *bool_matcher_results = NULL;
bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr);
int bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, pattern_set_size,
bool_matcher_results, hs_instance->n_expr);
if (bool_matcher_ret < 0) {
ret = -1;
goto next;
}
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
results[matched_index] = bool_matcher_results[matched_index].expr_id;
results[matched_index].item_id = bool_matcher_results[matched_index].expr_id;
results[matched_index].user_tag = bool_matcher_results[matched_index].user_tag;
}
*n_results = bool_matcher_ret;
next:
FREE(bool_matcher_results);
utarray_free(pattern_id_set);
return 0;
return ret;
}
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id)
@@ -487,14 +509,16 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
return hs_stream;
}
int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len,
int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream,
const char *data, size_t data_len,
int results[], size_t *n_results)
{
hs_error_t err;
int thread_id = hs_stream->thread_id;
if (hs_stream->literal_stream != NULL) {
err = hs_scan_stream(hs_stream->literal_stream, data, data_len, 0, hs_stream->hs_rt->scratchs[thread_id],
err = hs_scan_stream(hs_stream->literal_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id],
matched_event_cb, hs_stream->pattern_id_set);
if (err != HS_SUCCESS) {
//log_error()
@@ -503,7 +527,8 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
}
if (hs_stream->regex_stream != NULL) {
err = hs_scan_stream(hs_stream->regex_stream, data, data_len, 0, hs_stream->hs_rt->scratchs[thread_id],
err = hs_scan_stream(hs_stream->regex_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id],
matched_event_cb, hs_stream->pattern_id_set);
if (err != HS_SUCCESS) {
//log_error()
@@ -519,8 +544,10 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
}
size_t matched_index = 0;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_stream->n_expr);
struct bool_expr_match *bool_matcher_results = NULL;
bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size,
bool_matcher_results, hs_stream->n_expr);
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
results[matched_index] = bool_matcher_results[matched_index].expr_id;
}
@@ -533,10 +560,19 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
{
if (NULL == hs_stream) {
return;
}
int thread_id = hs_stream->thread_id;
hs_close_stream(hs_stream->literal_stream, hs_stream->hs_rt->scratchs[thread_id], NULL, NULL);
hs_close_stream(hs_stream->regex_stream, hs_stream->hs_rt->scratchs[thread_id], NULL, NULL);
if (hs_stream->hs_rt != NULL) {
hs_close_stream(hs_stream->literal_stream,
hs_stream->hs_rt->scratchs[thread_id], NULL, NULL);
hs_close_stream(hs_stream->regex_stream,
hs_stream->hs_rt->scratchs[thread_id], NULL, NULL);
}
utarray_free(hs_stream->pattern_id_set);
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */

11
scanner/adapter_hs.h
View File

@@ -38,6 +38,11 @@ enum pattern_type {
PATTERN_TYPE_REG,
};
struct hs_scan_result {
int item_id;
void *user_tag;
};
typedef struct {
/* pattern type */
int type;
@@ -53,6 +58,7 @@ typedef struct {
uint32_t expr_id;
size_t n_patterns;
scan_pattern_t patterns[MAX_EXPR_PATTERN_NUM];
void *user_tag;
} and_expr_t;
/**
@@ -65,7 +71,8 @@ typedef struct {
*
* @retval the pointer to adapter_hs instance
*/
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array, struct log_handle *logger);
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array,
size_t n_expr_array, struct log_handle *logger);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id
@@ -78,7 +85,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
* @param n_results: number of elements in array of expr_id
*/
int adapter_hs_scan(struct adapter_hs *instance, int thread_id, const char *data, size_t data_len,
int results[], size_t *n_results);
struct hs_scan_result *results, size_t *n_results);
/**
* @brief destroy adapter_hs instance

6
scanner/adapter_hs_gtest.cpp
View File

@@ -1,6 +1,5 @@
#include <gtest/gtest.h>
#include "utils.h"
#include "adapter_hs.h"
int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr)
@@ -33,7 +32,7 @@ int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr)
char *sub_pattern_token = strtok_r(pattern_token, ":", &save_pattern_ptr);
expr[i].patterns[j].type = atoi(sub_pattern_token);
size_t str_len = strlen(save_pattern_ptr);
expr[i].patterns[j].pat = ALLOC(char, str_len);
expr[i].patterns[j].pat = (char *)calloc(sizeof(char), str_len);
memcpy(expr[i].patterns[j].pat, save_pattern_ptr, str_len);
expr[i].patterns[j].pat_len = str_len;
j++;
@@ -50,7 +49,8 @@ void expr_array_free(and_expr_t expr_array[], size_t n_expr_array)
{
for (size_t i = 0; i < n_expr_array; i++) {
for (size_t j = 0; j < expr_array[i].n_patterns; j++) {
FREE(expr_array[i].patterns[j].pat);
free(expr_array[i].patterns[j].pat);
expr_array[i].patterns[j].pat = NULL;
}
}
}

5
src/inc_internal/json2iris.h
View File

@@ -18,8 +18,9 @@ extern "C"
#include "hiredis/hiredis.h"
int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx,
char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo,
int json2iris(const char *json_buff, const char *json_filename,
redisContext *redis_write_ctx, char *iris_dir_buf,
int buf_len, char *encrypt_key, char *encrypt_algo,
struct log_handle *logger);
#ifdef __cpluscplus

43
src/inc_internal/maat_compile.h
View File

@@ -33,14 +33,17 @@ struct compile_ex_data_schema {
};
/* compile schema API */
void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void compile_schema_free(void *compile_schema);
void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void group2compile_schema_free(void *g2c_schema);
int group2compile_associated_compile_table_id(void *g2c_schema);
int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id,
int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema,
int table_id,
maat_rule_ex_new_func_t *new_func,
maat_rule_ex_free_func_t *free_func,
maat_rule_ex_dup_func_t *dup_func,
@@ -52,37 +55,43 @@ compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, siz
size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema);
/* compile runtime API */
void *compile_runtime_new(void *compile_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
void *compile_runtime_new(void *compile_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void compile_runtime_free(void *compile_runtime);
int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line,
int valid_column);
int compile_runtime_update(void *compile_runtime, void *compile_schema,
const char *line, int valid_column);
int compile_runtime_commit(void *compile_runtime);
int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids,
int vt_id, int *compile_ids, size_t compile_ids_size, struct maat_state *state);
int compile_runtime_match(struct compile_runtime *compile_rt, int *compile_ids,
size_t compile_ids_size, struct maat_state *state);
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct group2group_runtime *g2g_rt,
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
struct group2group_runtime *g2g_rt,
struct maat_compile_state *compile_state,
struct maat_hit_path *hit_paths, size_t hit_path_siz);
struct maat_hit_path *hit_paths,
size_t n_hit_path);
/* group2compile runtime API */
void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
void *group2compile_runtime_new(void *g2c_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime);
void group2compile_runtime_free(void *g2c_runtime);
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line,
int valid_column);
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
const char *line, int valid_column);
/* maat compile state API */
struct maat_compile_state;
struct maat_compile_state *maat_compile_state_new(int thread_id);
void maat_compile_state_free(struct maat_compile_state *compile_state);
void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id,
int Nth_scan, int Nth_item_result);
void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, void *compile_runtime,
int group_id, int virtual_table_id);
int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
int *hit_item_ids, size_t hit_item_cnt,
int *group_ids, size_t group_ids_size,
size_t *n_hit_group_id, struct maat_state *state);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);
#ifdef __cpluscplus

7
src/inc_internal/maat_config_monitor.h
View File

@@ -18,17 +18,14 @@ extern "C"
#include <stdint.h>
#define CM_UPDATE_TYPE_NONE 0
#define CM_UPDATE_TYPE_FULL 1
#define CM_UPDATE_TYPE_INC 2
void config_monitor_traverse(long long version, const char *idx_dir,
void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *), void *u_param,
struct log_handle *logger);
int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz);
int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
char *err_str, size_t err_str_sz);
#ifdef __cpluscplus
}

28
src/inc_internal/maat_ex_data.h
View File

@@ -33,11 +33,10 @@ struct ex_container_ctx {
struct ex_data_runtime;
/* ex_data_runtime API */
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn,
struct log_handle *logger);
struct ex_data_runtime *
ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn,
struct log_handle *logger);
void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt);
void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt);
/* ex_data_runtime cache row API */
@@ -56,19 +55,22 @@ struct ex_data_schema *ex_data_schema_new(maat_plugin_ex_new_func_t *new_func,
long argl, void *argp);
void ex_data_schema_free(struct ex_data_schema *ex_schema);
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema);
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt,
struct ex_data_schema *schema);
/* set user_ctx API */
void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, struct ex_container_ctx *container_ctx);
void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt,
struct ex_container_ctx *container_ctx);
struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt);
struct ex_container_ctx *
ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt);
struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data);
void ex_data_container_free(void *ctx, void *data);
/* ex_data_runtime ex data API */
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt,
const char *row, const char *key, size_t key_len);
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row,
const char *key, size_t key_len);
int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len,
@@ -80,14 +82,14 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container ***ex_container);
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt);
int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt);
#ifdef __cpluscplus
}
#endif

24
src/inc_internal/maat_expr.h
View File

@@ -21,16 +21,18 @@ extern "C"
struct expr_runtime;
void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void expr_schema_free(void *expr_schema);
/* expr runtime API */
void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
void *expr_runtime_new(void *expr_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void expr_runtime_free(void *expr_runtime);
int expr_runtime_updating_flag(void *expr_runtime);
int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column);
int expr_runtime_update(void *expr_runtime, void *expr_schema,
const char *line, int valid_column);
int expr_runtime_commit(void *expr_runtime);
/* expr runtime scan API */
@@ -39,15 +41,19 @@ int expr_runtime_commit(void *expr_runtime);
*
* @retval the num of hit group_id
*/
int expr_runtime_scan_string(struct expr_runtime *expr_runtime, int thread_id,
const char *data, size_t data_len, int group_ids[],
size_t max_hit_num, int virtual_table_id,
struct maat_state *state);
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, int results[], size_t *n_result);
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data,
size_t data_len, int result[], size_t *n_result);
void expr_runtime_stream_close(struct expr_runtime *expr_rt);
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id);
long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread);
#ifdef __cpluscplus
}
#endif

3
src/inc_internal/maat_garbage_collection.h
View File

@@ -24,7 +24,8 @@ struct maat_garbage_bin *maat_garbage_bin_new(int default_timeout);
void maat_garbage_bin_free(struct maat_garbage_bin *bin);
void maat_garbage_bagging(struct maat_garbage_bin *bin, void *garbage, void (* func)(void *));
void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage,
void (* func)(void *));
void maat_garbage_collect_routine(struct maat_garbage_bin *bin);

53
src/inc_internal/maat_group.h
View File

@@ -21,63 +21,34 @@ extern "C"
#include "igraph/igraph.h"
#include "maat_kv.h"
struct maat_group {
igraph_integer_t vertex_id;
int group_id;
int ref_by_compile_cnt;
int ref_by_superior_group_cnt;
int ref_by_subordinate_group_cnt;
size_t top_group_cnt;
int *top_group_ids;
UT_hash_handle hh_group_id;
UT_hash_handle hh_vertex_id;
};
/* maat group topology API */
struct maat_group_topology;
/**
* @retval if not found, return NULL
*/
//struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id);
//struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id);
//void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group);
//int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
//int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
/* build top groups */
//int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo);
struct maat_group;
/* group2group schema API */
void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void group2group_schema_free(void *g2g_schema);
/* group2group runtime API */
void *group2group_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
void *group2group_runtime_new(void *g2g_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void group2group_runtime_free(void *g2g_runtime);
void maat_group_ref_inc(struct maat_group *group);
void maat_group_ref_dec(struct maat_group *group);
struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id);
void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group);
struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id);
int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int superior_group_id);
int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, int superior_group_id);
int group2group_runtime_build_top_groups(void *g2g_runtime);
int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids,
size_t n_group_ids, int *top_group_ids);
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column);
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
const char *line, int valid_column);
int group2group_runtime_commit(void *g2g_runtime);
int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t n_group_ids,
int *top_group_ids);
#ifdef __cpluscplus
}
#endif

36
src/inc_internal/maat_ip.h
View File

@@ -16,30 +16,34 @@ extern "C"
{
#endif
struct ip_plus_runtime;
struct ip_runtime;
void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void ip_plus_schema_free(void *ip_plus_schema);
void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void ip_schema_free(void *ip_schema);
/* ip plus runtime API */
void *ip_plus_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void ip_plus_runtime_free(void *ip_plus_runtime);
/* ip runtime API */
void *ip_runtime_new(void *ip_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void ip_runtime_free(void *ip_runtime);
int ip_plus_runtime_updating_flag(void *ip_plus_runtime);
int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line,
int valid_column);
int ip_plus_runtime_commit(void *ip_plus_runtime);
int ip_runtime_update(void *ip_runtime, void *ip_schema,
const char *line, int valid_column);
int ip_runtime_commit(void *ip_runtime);
struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt);
struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt);
/* ip runtime scan API */
int ip_plus_runtime_scan_ipv4(struct ip_plus_runtime *ip_plus_rt, int thread_id, uint32_t ip_addr,
int *group_id_array, size_t n_group_id_array, int virtual_table_id,
struct maat_state *state);
int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, int ip_type,
uint8_t *ip_addr, int *group_ids, size_t group_id_size,
int vtable_id, struct maat_state *state);
void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id);
long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread);
#ifdef __cpluscplus
}
#endif
#endif
#endif

11
src/inc_internal/maat_ip_plugin.h
View File

@@ -23,7 +23,8 @@ extern "C"
struct ip_plugin_runtime;
/* ip plugin schema API */
void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void ip_plugin_schema_free(void *ip_plugin_schema);
/* ip plugin table ex data API */
@@ -37,13 +38,13 @@ int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
struct log_handle *logger);
/* ip plugin runtime API */
void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void ip_plugin_runtime_free(void *ip_plugin_runtime);
int ip_plugin_runtime_updating_flag(void *ip_plugin_runtime);
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line,
int valid_column);
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
const char *line, int valid_column);
int ip_plugin_runtime_commit(void *ip_plugin_runtime);
struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime);

24
src/inc_internal/maat_plugin.h
View File

@@ -21,7 +21,8 @@ extern "C"
#define MAX_FOREIGN_CLMN_NUM 8
/* plugin schema API */
void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void plugin_schema_free(void *plugin_schema);
struct plugin_schema;
@@ -38,23 +39,22 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema);
int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns);
/* plugin table ex data API */
int plugin_table_set_ex_data_schema(void *custom_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger);
int plugin_table_set_ex_data_schema(void *plugin_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger);
struct ex_data_schema *plugin_table_get_ex_data_schema(void *custom_schema);
/* plugin runtime API */
void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
void *plugin_runtime_new(void *plugin_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void plugin_runtime_free(void *plugin_runtime);
int plugin_runtime_updating_flag(void *plugin_runtime);
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line,
int valid_column);
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
const char *line, int valid_column);
int plugin_runtime_commit(void *plugin_runtime);
struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime);

2
src/inc_internal/maat_redis_monitor.h
View File

@@ -20,7 +20,7 @@ extern "C"
#include <stdint.h>
void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx,
void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *), void *u_param);

65
src/inc_internal/maat_rule.h
View File

@@ -34,6 +34,22 @@ extern "C"
#define MAX_TABLE_NUM 256
#define MAAT_UPDATE_TYPE_NONE 0
#define MAAT_UPDATE_TYPE_FULL 1
#define MAAT_UPDATE_TYPE_INC 2
enum last_scan_flag {
LAST_SCAN_UNSET,
LAST_SCAN_SET,
LAST_SCAN_FINISHED
};
enum tag_match {
TAG_MATCH_ERR = -1,
TAG_MATCH_UNMATCHED,
TAG_MATCH_MATCHED
};
struct maat_rule_head {
int config_id;
int service_id;
@@ -89,7 +105,7 @@ struct compile_rule {
struct group2group_rule {
int group_id;
int superior_group_id;
int super_group_id;
};
struct maat_runtime {
@@ -223,9 +239,11 @@ struct maat {
/* statistics */
long long line_cmd_acc_num;
long long *outer_mid_cnt;
long long *compile_mid_cnt;
long long *outer_state_cnt;
long long *compile_state_cnt;
long long *thread_call_cnt;
long long *hit_cnt;
long long *not_grp_hit_cnt;
long long scan_err_cnt;
};
@@ -238,7 +256,7 @@ struct maat_state {
unsigned char is_last_scan;
int district_id; //-1: Any District; -2: Unkonwn District;
int scan_cnt;
struct maat_compile_state *compile_mid;
struct maat_compile_state *compile_state;
};
size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger);
@@ -264,7 +282,8 @@ void *rule_monitor_loop(void *arg);
void maat_read_full_config(struct maat *maat_instance);
/* maat command API for internal */
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db, struct log_handle *logger);
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger);
redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...);
@@ -274,35 +293,39 @@ long long maat_cmd_redis_server_time_s(redisContext *c);
long long maat_cmd_read_redis_integer(const redisReply *reply);
int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq);
int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
int valid_column_seq);
const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len);
int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num,
long long server_time, struct log_handle *logger);
int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
size_t serial_rule_num, long long server_time,
struct log_handle *logger);
void maat_cmd_clear_rule_cache(struct serial_rule *s_rule);
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version,
long long *new_version, struct table_manager* tbl_mgr,
struct serial_rule **list, int *update_type, int cumulative_off,
struct log_handle *logger);
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version,
long long desired_version, long long *new_version,
struct table_manager *tbl_mgr, struct serial_rule **list,
int *update_type, int cumulative_off, struct log_handle *logger);
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process,
struct log_handle *logger);
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_process, struct log_handle *logger);
int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num,
const char *dir, struct log_handle *logger);
int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, const char* dir, struct log_handle *logger);
void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn,
struct log_handle *logger);
void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_fn, struct log_handle *logger);
void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule);
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id,
const char *table_name, const char *line, long long timeout);
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
unsigned long rule_id, const char *table_name,
const char *line, long long timeout);
void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, const char *srv_def, int srv_def_len);
void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head,
const char *srv_def, int srv_def_len);
#ifdef __cpluscplus
}

16
src/inc_internal/maat_table.h
View File

@@ -43,8 +43,9 @@ enum table_type {
struct table_manager;
struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags,
struct log_handle *logger);
struct table_manager *
table_manager_create(const char *table_info_path, const char *accept_tags,
struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num,
struct maat_garbage_bin *garbage_bin);
void table_manager_runtime_destroy(struct table_manager *tbl_mgr);
@@ -53,14 +54,21 @@ void table_manager_destroy(struct table_manager *tbl_mgr);
size_t table_manager_table_count(struct table_manager *tbl_mgr);
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name);
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id);
int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr);
int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr);
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);
size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr);
int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags);
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str,
size_t district_str_len, int *district_id);
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);
int table_manager_runtime_updating_flag(struct table_manager *tbl_mgr, int table_id);
int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line);
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id);

21
src/inc_internal/maat_utils.h
View File

@@ -31,6 +31,27 @@ extern "C"
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
#define FREE(ptr) \
{ \
if (ptr) \
{ \
free(ptr); \
ptr = NULL; \
} \
}
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
#endif
#define MAX_SCANNER_HIT_COMPILE_NUM 4096
#define MAX_SCANNER_HIT_GROUP_NUM 4096
#define MAX_SCANNER_HIT_ITEM_NUM 4096

3
src/inc_internal/maat_virtual.h
View File

@@ -29,7 +29,8 @@ enum scan_type {
SCAN_TYPE_MAX
};
void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void virtual_schema_free(void *virtual_schema);

7
src/inc_internal/rcu_hash.h
View File

@@ -62,13 +62,6 @@ size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable);
size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array);
/**
* @brief check if rcu hash table is updating
*
* @retval 1 means htable is updating, otherwise 0
*/
int rcu_hash_updating_flag(struct rcu_hash_table *htable);
#ifdef __cpluscplus
}
#endif

221
src/json2iris.cpp
View File

@@ -15,7 +15,6 @@
#include <errno.h>
#include <sys/stat.h>
#include "utils.h"
#include "json2iris.h"
#include "cJSON/cJSON.h"
#include "maat_kv.h"
@@ -23,7 +22,7 @@
#include "maat_rule.h"
#include "uthash/uthash.h"
#define MODULE_JSON2IRIS module_name_str("maat.json2iris")
#define MODULE_JSON2IRIS module_name_str("maat.json2iris")
#define MAX_COLUMN_NUM 32
@@ -89,7 +88,9 @@ struct translate_command {
int default_int;
};
struct iris_table *query_table_info(struct iris_description *p_iris, const char *table_name, enum table_type table_type)
struct iris_table *query_table_info(struct iris_description *p_iris,
const char *table_name,
enum table_type table_type)
{
struct iris_table *table_info = NULL;
HASH_FIND(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info);
@@ -97,8 +98,10 @@ struct iris_table *query_table_info(struct iris_description *p_iris, const char
table_info = ALLOC(struct iris_table, 1);
table_info->line_count = 0;
table_info->table_type = table_type;
memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name) - 1, strlen(table_name)));
snprintf(table_info->table_path, sizeof(table_info->table_path), "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name);
memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name) - 1,
strlen(table_name)));
snprintf(table_info->table_path, sizeof(table_info->table_path),
"%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name);
HASH_ADD_KEYPTR(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info);
}
@@ -114,14 +117,18 @@ void free_iris_table_info(void *p)
FREE(table);
}
int set_iris_descriptor(const char *json_file, cJSON *json, const char *encrypt_key, const char *encrypt_algo,
const char *compile_tn, const char *group2compile_tn, const char* group2group_tn,
redisContext *redis_write_ctx, struct iris_description *iris_cfg)
int set_iris_descriptor(const char *json_file, cJSON *json,
const char *encrypt_key, const char *encrypt_algo,
const char *compile_tn, const char *group2compile_tn,
const char* group2group_tn, redisContext *redis_write_ctx,
struct iris_description *iris_cfg)
{
memset(iris_cfg, 0, sizeof(struct iris_description));
snprintf(iris_cfg->tmp_iris_dir, sizeof(iris_cfg->tmp_iris_dir), "%s_iris_tmp", json_file);
snprintf(iris_cfg->tmp_iris_index_dir, sizeof(iris_cfg->tmp_iris_index_dir), "%s_iris_tmp/index", json_file);
snprintf(iris_cfg->index_path, sizeof(iris_cfg->index_path), "%s/full_config_index.%010d", iris_cfg->tmp_iris_index_dir, json_version);
snprintf(iris_cfg->tmp_iris_index_dir, sizeof(iris_cfg->tmp_iris_index_dir),
"%s_iris_tmp/index", json_file);
snprintf(iris_cfg->index_path, sizeof(iris_cfg->index_path), "%s/full_config_index.%010d",
iris_cfg->tmp_iris_index_dir, json_version);
iris_cfg->redis_write_ctx = redis_write_ctx;
iris_cfg->str2int_map = maat_kv_store_new();
@@ -219,13 +226,15 @@ int create_tmp_dir(struct iris_description *p)
return 0;
}
int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_description *p_iris,
int write_plugin_line(cJSON *plug_table_json, int sequence,
struct iris_description *p_iris,
struct log_handle *logger)
{
cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_JSON2IRIS,
"The %d plugin_table's table_name not defined or format error", sequence);
"The %d plugin_table's table_name not defined or format error",
sequence);
return -1;
}
const char *table_name = item->valuestring;
@@ -233,7 +242,8 @@ int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_descript
cJSON *table_content = cJSON_GetObjectItem(plug_table_json, "table_content");
if (NULL == table_content || table_content->type != cJSON_Array) {
log_error(logger, MODULE_JSON2IRIS,
"%d plugin_table's table_content not defined or format error", sequence);
"%d plugin_table's table_content not defined or format error",
sequence);
return -1;
}
int line_cnt = cJSON_GetArraySize(table_content);
@@ -244,21 +254,25 @@ int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_descript
for (int i = 0; i < line_cnt; i++) {
each_line = cJSON_GetArrayItem(table_content, i);
if (NULL == each_line || each_line->type != cJSON_String) {
log_error(logger, MODULE_JSON2IRIS, "plugin_table %s's line %d format error",
table_info->table_name, i + 1);
log_error(logger, MODULE_JSON2IRIS,
"plugin_table %s's line %d format error",
table_info->table_name, i + 1);
continue;
}
line_content = each_line->valuestring;
table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), line_content, strlen(line_content));
table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), "\n", 1);
table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos,
&(table_info->buff_sz), line_content, strlen(line_content));
table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos,
&(table_info->buff_sz), "\n", 1);
table_info->line_count++;
}
return 0;
}
static struct group_info *group_info_read(struct group_info *group_name_map, const char *group_name)
static struct group_info *group_info_read(struct group_info *group_name_map,
const char *group_name)
{
struct group_info *node = NULL;
HASH_FIND(hh, group_name_map, group_name, strlen(group_name), node);
@@ -274,7 +288,8 @@ static int get_group_seq(struct iris_description *iris_cfg)
if (NULL == iris_cfg->redis_write_ctx) {
sequence = iris_cfg->group_cnt;
} else {
data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx, "INCRBY %s 1", mr_group_id_var);
data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx,
"INCRBY %s 1", mr_group_id_var);
sequence = (int)data_reply->integer - 1;
freeReplyObject(data_reply);
data_reply = NULL;
@@ -284,7 +299,8 @@ static int get_group_seq(struct iris_description *iris_cfg)
return sequence;
}
static struct group_info *group_info_add_unsafe(struct iris_description *p_iris, const char *group_name)
static struct group_info *group_info_add_unsafe(struct iris_description *p_iris,
const char *group_name)
{
static struct group_info untitled_group;
struct group_info *group_info = NULL;
@@ -308,7 +324,8 @@ static int get_region_seq(struct iris_description *iris_cfg)
if (NULL == iris_cfg->redis_write_ctx) {
sequence = iris_cfg->region_cnt;
} else {
redisReply *data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx, "INCRBY %s 1", mr_region_id_var);
redisReply *data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx,
"INCRBY %s 1", mr_region_id_var);
sequence = (int)data_reply->integer - 1;
freeReplyObject(data_reply);
}
@@ -317,7 +334,8 @@ static int get_region_seq(struct iris_description *iris_cfg)
return sequence;
}
int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct translate_command *cmd, int cmd_cnt,
int direct_write_rule(cJSON *json, struct maat_kv_store *str2int,
struct translate_command *cmd, int cmd_cnt,
struct iris_table *table, struct log_handle *logger)
{
int i = 0;
@@ -335,7 +353,8 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla
}
if (NULL == item || item->type != cmd[i].json_type) {
log_error(logger, MODULE_JSON2IRIS, "%s not defined or wrong format", cmd[i].json_string);
log_error(logger, MODULE_JSON2IRIS,
"%s not defined or wrong format", cmd[i].json_string);
ret = -1;
goto error_out;
}
@@ -345,7 +364,8 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla
char *p = item->valuestring;
ret = maat_kv_read(str2int, p, &int_value);
if (ret < 0) {
log_error(logger, MODULE_JSON2IRIS, "%s's value %s is not valid format", cmd[i].json_string, p);
log_error(logger, MODULE_JSON2IRIS,
"%s's value %s is not valid format", cmd[i].json_string, p);
FREE(p);
ret = -1;
goto error_out;
@@ -369,7 +389,8 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla
}
for (i = 0; i < cmd_cnt; i++) {
table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, cmd[i].json_value, strlen(cmd[i].json_value));
table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz,
cmd[i].json_value, strlen(cmd[i].json_value));
table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1);
}
table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1);
@@ -384,8 +405,8 @@ error_out:
return ret;
}
int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
int write_expr_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
@@ -428,11 +449,12 @@ int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
}
int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
@@ -540,11 +562,12 @@ int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, stru
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
}
int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
int write_intval_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
@@ -576,11 +599,12 @@ int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struc
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
}
int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
int write_digest_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
@@ -610,11 +634,12 @@ int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struc
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
}
int write_similar_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
int write_similar_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
@@ -640,16 +665,18 @@ int write_similar_line(cJSON *region_json, struct iris_description *p_iris, stru
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
}
int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct iris_description *p_iris,
struct log_handle *logger)
int write_region_rule(cJSON *region_json, int compile_id, int group_id,
struct iris_description *p_iris, struct log_handle *logger)
{
cJSON *item = cJSON_GetObjectItem(region_json, "table_name");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_JSON2IRIS, "compile rule %d's table_name not defined or format error",
compile_id);
log_error(logger, MODULE_JSON2IRIS,
"compile rule %d's table_name not defined or format error",
compile_id);
return -1;
}
const char *table_name = item->valuestring;
@@ -657,7 +684,8 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
item = cJSON_GetObjectItem(region_json, "table_type");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_JSON2IRIS,
"compile rule %d's table name %s's table_type not defined or format error", compile_id, table_name);
"compile rule %d's table name %s's table_type not defined or format error",
compile_id, table_name);
return -1;
}
@@ -665,15 +693,17 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
enum table_type table_type = TABLE_TYPE_EXPR;
int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type));
if (ret != 1) {
log_error(logger, MODULE_JSON2IRIS, "compile rule %d table name %s's table_type %s invalid",
compile_id, table_name, table_type_str);
log_error(logger, MODULE_JSON2IRIS,
"compile rule %d table name %s's table_type %s invalid",
compile_id, table_name, table_type_str);
return -1;
}
cJSON *table_content = cJSON_GetObjectItem(region_json, "table_content");
if (NULL == table_content || table_content->type != cJSON_Object) {
log_error(logger, MODULE_JSON2IRIS,
"compile rule %d table name %s's table_content not defined or format error", compile_id, table_name);
"compile rule %d table name %s's table_content not defined or format error",
compile_id, table_name);
return -1;
}
@@ -709,33 +739,41 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
return ret;
}
int write_group2compile_line(int group_id, int compile_id, int group_not_flag, int clause_index,
const char *virtual_table, struct iris_description *p_iris)
int write_group2compile_line(int group_id, int compile_id, int group_not_flag,
int clause_index, const char *vtable,
struct iris_description *p_iris)
{
char buff[4096] = {0};
struct iris_table *table = p_iris->group2compile_table;
snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id, group_not_flag, virtual_table, clause_index);
table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff));
snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id,
group_not_flag, vtable, clause_index);
table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz),
buff, strlen(buff));
table->line_count++;
return 0;
}
int write_group2group_line(int group_id, int superior_group_id, struct iris_description *p_iris)
int write_group2group_line(int group_id, int super_group_id,
struct iris_description *p_iris)
{
char buff[4096] = {0};
struct iris_table *table = p_iris->group2group_table;
snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id, superior_group_id);
table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff));
snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id,
super_group_id);
table->write_pos += memcat(&(table->buff), table->write_pos,
&(table->buff_sz), buff, strlen(buff));
table->line_count++;
return 0;
}
int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id,
int Nth_group, struct iris_description *p_iris, struct log_handle *logger)
int write_group_rule(cJSON *group_json, int parent_id,
int parent_type, int tracking_compile_id,
int Nth_group, struct iris_description *p_iris,
struct log_handle *logger)
{
int ret = 0;
int group_not_flag = 0;
@@ -784,10 +822,11 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
if (region_json != NULL) {
cJSON *region_rule = NULL;
cJSON_ArrayForEach(region_rule, region_json) {
ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris, logger);
ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id,
p_iris, logger);
if (ret < 0) {
log_error(logger, MODULE_JSON2IRIS, "compile rule %d write region error",
tracking_compile_id);
log_error(logger, MODULE_JSON2IRIS,
"compile rule %d write region error", tracking_compile_id);
return -1;
}
}
@@ -799,7 +838,8 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
int i = 0;
cJSON_ArrayForEach(item, sub_groups) {
i++;
ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris, logger);
ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP,
tracking_compile_id, i, p_iris, logger);
if (ret < 0) {
return -1;
}
@@ -814,20 +854,23 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
}
if (parent_type == PARENT_TYPE_COMPILE) {
ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag, clause_index, virtual_table, p_iris);
ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag,
clause_index, virtual_table, p_iris);
} else {
ret = write_group2group_line(group_info->group_id, parent_id, p_iris);
}
if (ret < 0) {
log_error(logger, MODULE_JSON2IRIS, "%s rule %d write group error", str_parent_type[parent_type], parent_id);
log_error(logger, MODULE_JSON2IRIS,
"%s rule %d write group error", str_parent_type[parent_type], parent_id);
return -1;
}
return 0;
}
int write_compile_line(cJSON *compile, struct iris_description *p_iris, struct log_handle *logger)
int write_compile_line(cJSON *compile, struct iris_description *p_iris,
struct log_handle *logger)
{
cJSON *item=cJSON_GetObjectItem(compile, "compile_id");
if (item->type != cJSON_Number) {
@@ -925,7 +968,8 @@ int write_compile_line(cJSON *compile, struct iris_description *p_iris, struct l
table_info = query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE);
}
int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt, table_info, logger);
int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt,
table_info, logger);
if (ret < 0) {
return -1;
}
@@ -949,15 +993,18 @@ void write_table_idx(struct iris_description *p_iris, struct iris_table *table)
if (p_iris->encrypt_key) {
unsigned char *encrypt_buff = NULL;
size_t encrypt_buff_sz = 0;
int ret = crypt_memory(buff, table_file_sz, &encrypt_buff, &encrypt_buff_sz, p_iris->encrypt_key,
int ret = crypt_memory(buff, table_file_sz, &encrypt_buff,
&encrypt_buff_sz, p_iris->encrypt_key,
p_iris->encrypt_algo, 1, err_str, sizeof(err_str));
assert(ret == 0);
fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp);
fprintf(p_iris->idx_fp, "%s\t%d\t%s\t%s\n", table->table_name, table->line_count, table->table_path, p_iris->encrypt_algo);
fprintf(p_iris->idx_fp, "%s\t%d\t%s\t%s\n", table->table_name, table->line_count,
table->table_path, p_iris->encrypt_algo);
FREE(encrypt_buff);
} else {
fwrite(buff, table_file_sz, 1, table_fp);
fprintf(p_iris->idx_fp, "%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path);
fprintf(p_iris->idx_fp, "%s\t%d\t%s\n", table->table_name, table->line_count,
table->table_path);
}
fclose(table_fp);
@@ -968,8 +1015,9 @@ int write_index_file(struct iris_description *p_iris, struct log_handle *logger)
{
p_iris->idx_fp = fopen(p_iris->index_path, "w");
if (NULL == p_iris->idx_fp) {
log_error(logger, MODULE_JSON2IRIS, "index file %s fopen error %s",
p_iris->index_path, strerror(errno));
log_error(logger, MODULE_JSON2IRIS,
"index file %s fopen error %s",
p_iris->index_path, strerror(errno));
return -1;
}
@@ -985,7 +1033,8 @@ int write_index_file(struct iris_description *p_iris, struct log_handle *logger)
return 0;
}
int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *logger)
int write_iris(cJSON *json, struct iris_description *p_iris,
struct log_handle *logger)
{
int i=0;
int ret=0;
@@ -1017,7 +1066,8 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *
parent_group = group_info_add_unsafe(p_iris, parent_group_name);
}
ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris, logger);
ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP,
0, 0, p_iris, logger);
if (ret < 0) {
return -1;
}
@@ -1041,20 +1091,23 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *
group_array = cJSON_GetObjectItem(compile_obj, "groups");
if (NULL == group_array) {
log_error(logger, MODULE_JSON2IRIS, "compile rule %d have no group", compile_id);
log_error(logger, MODULE_JSON2IRIS,
"compile rule %d have no group", compile_id);
return -1;
}
int group_cnt = cJSON_GetArraySize(group_array);
if (group_cnt <= 0) {
log_error(logger, MODULE_JSON2IRIS, "compile rule %d have no groups", compile_id);
log_error(logger, MODULE_JSON2IRIS,
"compile rule %d have no groups", compile_id);
return -1;
}
i = 0;
cJSON *group_obj = NULL;
cJSON_ArrayForEach(group_obj, group_array) {
ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, compile_id, i, p_iris, logger);
ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE,
compile_id, i, p_iris, logger);
if (ret < 0) {
return -1;
}
@@ -1070,8 +1123,9 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *
return 0;
}
int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx,
char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo,
int json2iris(const char *json_buff, const char *json_filename,
redisContext *redis_write_ctx, char *iris_dir_buf,
int buf_len, char *encrypt_key, char *encrypt_algo,
struct log_handle *logger)
{
int ret = -1;
@@ -1085,7 +1139,8 @@ int json2iris(const char *json_buff, const char *json_filename, redisContext *re
cJSON *json = cJSON_Parse(json_buff);
if (!json) {
log_error(logger, MODULE_JSON2IRIS, "Error before: %-200.200s", cJSON_GetErrorPtr());
log_error(logger, MODULE_JSON2IRIS,
"Error before: %-200.200s", cJSON_GetErrorPtr());
goto error_out;
}
@@ -1105,15 +1160,16 @@ int json2iris(const char *json_buff, const char *json_filename, redisContext *re
}
ret = set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo,
compile_tbl_name, group2compile_tbl_name, group2group_tbl_name,
redis_write_ctx, &iris_cfg);
compile_tbl_name, group2compile_tbl_name,
group2group_tbl_name, redis_write_ctx, &iris_cfg);
if (ret < 0) {
goto error_out;
}
ret = create_tmp_dir(&iris_cfg);
if (ret < 0) {
log_error(logger, MODULE_JSON2IRIS, "create tmp folder %s error", iris_cfg.tmp_iris_dir);
log_error(logger, MODULE_JSON2IRIS,
"create tmp folder %s error", iris_cfg.tmp_iris_dir);
goto error_out;
}
@@ -1121,7 +1177,8 @@ int json2iris(const char *json_buff, const char *json_filename, redisContext *re
if (ret < 0) {
goto error_out;
}
memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir) + 1, (unsigned int)buf_len));
memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir,
MIN(strlen(iris_cfg.tmp_iris_index_dir) + 1, (unsigned int)buf_len));
cJSON_Delete(json);
clear_iris_descriptor(&iris_cfg);

469
src/maat_api.cpp
View File

@@ -12,7 +12,6 @@
#include <string.h>
#include <assert.h>
#include "utils.h"
#include "maat_utils.h"
#include "json2iris.h"
#include "maat/maat.h"
@@ -38,15 +37,10 @@
#define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2
inline int scan_state_should_compile_NOT(struct maat_state *mid)
{
if (mid && mid->is_last_scan==1 && mid->compile_mid &&
maat_compile_state_has_NOT_clause(mid->compile_mid)) {
return 1;
} else {
return 0;
}
}
enum district_set_flag {
DISTRICT_FLAG_UNSET,
DISTRICT_FLAG_SET
};
struct maat_options* maat_options_new(void)
{
@@ -111,7 +105,8 @@ int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms)
return 0;
}
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len)
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name,
size_t name_len)
{
memcpy(opts->instance_name, instance_name, name_len);
@@ -125,7 +120,8 @@ int maat_options_set_deferred_load_on(struct maat_options *opts)
return 0;
}
int maat_options_set_iris(struct maat_options *opts, const char *full_directory, const char *increment_directory)
int maat_options_set_iris(struct maat_options *opts, const char *full_directory,
const char *increment_directory)
{
if (strlen(full_directory) >= NAME_MAX || strlen(increment_directory) >= NAME_MAX) {
return -1;
@@ -146,7 +142,8 @@ int maat_options_set_json_file(struct maat_options *opts, const char *json_filen
return 0;
}
int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, uint16_t redis_port, int redis_db)
int maat_options_set_redis(struct maat_options *opts, const char *redis_ip,
uint16_t redis_port, int redis_db)
{
memcpy(opts->redis_ctx.redis_ip, redis_ip, strlen(redis_ip));
opts->redis_ctx.redis_port = redis_port;
@@ -172,10 +169,13 @@ void maat_read_full_config(struct maat *maat_instance)
switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS:
mr_ctx = &(maat_instance->mr_ctx);
log_info(maat_instance->logger, MODULE_MAAT_API, "Maat initiate from Redis %s:%hu db%d",
log_info(maat_instance->logger, MODULE_MAAT_API,
"Maat initiate from Redis %s:%hu db%d",
mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port,
mr_ctx->redis_db, maat_instance->logger);
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip,
mr_ctx->redis_port,
mr_ctx->redis_db,
maat_instance->logger);
if (mr_ctx->read_ctx != NULL) {
redis_monitor_traverse(maat_instance->maat_version, mr_ctx,
maat_start_cb, maat_update_cb, maat_finish_cb,
@@ -200,9 +200,11 @@ void maat_read_full_config(struct maat *maat_instance)
}
break;
case DATA_SOURCE_JSON_FILE:
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str));
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file,
err_str, sizeof(err_str));
if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_API, "Maat re-initiate with JSON file %s failed: %s",
log_error(maat_instance->logger, MODULE_MAAT_API,
"Maat re-initiate with JSON file %s failed: %s",
maat_instance->json_ctx.json_file, err_str);
return;
}
@@ -212,7 +214,8 @@ void maat_read_full_config(struct maat *maat_instance)
maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance, maat_instance->logger);
if (NULL == maat_instance->creating_maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_API, "At initiation: NO effective rule in %s",
log_error(maat_instance->logger, MODULE_MAAT_API,
"At initiation: NO effective rule in %s",
maat_instance->json_ctx.iris_file);
}
break;
@@ -243,36 +246,44 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
} else {
char log_path[1024] = {0};
if (strlen(maat_instance->instance_name) > 0) {
snprintf(log_path, sizeof(log_path), "%s.log", maat_instance->instance_name);
snprintf(log_path, sizeof(log_path), "%s.log",
maat_instance->instance_name);
} else {
snprintf(log_path, sizeof(log_path), "maat.log");
}
maat_instance->logger = log_handle_create(log_path, opts->log_level);
}
maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags, maat_instance->logger);
maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags,
maat_instance->garbage_bin, maat_instance->logger);
if (NULL == maat_instance->tbl_mgr) {
goto failed;
}
maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
maat_instance->input_mode = opts->input_mode;
switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS:
memcpy(maat_instance->mr_ctx.redis_ip, opts->redis_ctx.redis_ip, strlen(opts->redis_ctx.redis_ip));
memcpy(maat_instance->mr_ctx.redis_ip, opts->redis_ctx.redis_ip,
strlen(opts->redis_ctx.redis_ip));
maat_instance->mr_ctx.redis_port = opts->redis_ctx.redis_port;
maat_instance->mr_ctx.redis_db = opts->redis_ctx.redis_db;
break;
case DATA_SOURCE_IRIS_FILE:
memcpy(maat_instance->iris_ctx.full_idx_dir, opts->iris_ctx.full_idx_dir, strlen(opts->iris_ctx.full_idx_dir));
memcpy(maat_instance->iris_ctx.inc_idx_dir, opts->iris_ctx.inc_idx_dir, strlen(opts->iris_ctx.inc_idx_dir));
memcpy(maat_instance->iris_ctx.full_idx_dir, opts->iris_ctx.full_idx_dir,
strlen(opts->iris_ctx.full_idx_dir));
memcpy(maat_instance->iris_ctx.inc_idx_dir, opts->iris_ctx.inc_idx_dir,
strlen(opts->iris_ctx.inc_idx_dir));
break;
case DATA_SOURCE_JSON_FILE:
memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file));
memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file,
strlen(opts->json_ctx.json_file));
break;
default:
log_error(maat_instance->logger, MODULE_MAAT_API, "data source unsupported:%d", maat_instance->input_mode);
log_error(maat_instance->logger, MODULE_MAAT_API,
"data source unsupported:%d", maat_instance->input_mode);
goto failed;
}
@@ -287,9 +298,11 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
(maat_instance->gc_timeout_ms / 1000);
maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s);
maat_instance->outer_mid_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->compile_mid_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->outer_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->compile_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->hit_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->not_grp_hit_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
pthread_mutex_init(&(maat_instance->background_update_mutex), NULL);
@@ -349,7 +362,8 @@ size_t generic_plugin_runtime_cached_row_count(void *custom_rt, enum table_type
return 0;
}
const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type, size_t row_id)
const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type,
size_t row_id)
{
return NULL;
}
@@ -383,7 +397,7 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id,
if (row_cnt > 0) {
if (start != NULL) {
start(MAAT_RULE_UPDATE_TYPE_FULL, u_para);
start(MAAT_UPDATE_TYPE_FULL, u_para);
}
for (size_t i = 0; i < row_cnt; i++) {
@@ -457,8 +471,9 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t
return 0;
}
void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, int table_id,
enum table_type table_type, int valid_column)
void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema,
int table_id, enum table_type table_type,
int valid_column)
{
struct ex_data_schema *ex_data_schema = NULL;
struct ex_data_runtime *ex_data_rt = NULL;
@@ -518,26 +533,32 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i
{
pthread_mutex_lock(&(maat_instance->background_update_mutex));
int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id,
new_func, free_func, dup_func, argl, argp,
maat_instance->logger);
new_func, free_func, dup_func,
argl, argp, maat_instance->logger);
if (ret < 0) {
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return -1;
}
enum table_type table_type = TABLE_TYPE_INVALID;
int valid_column = -1;
if (maat_instance->maat_rt != NULL) {
void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id);
generic_plugin_runtime_commit_ex_schema(runtime, schema, table_id, table_type, valid_column);
assert(runtime != NULL && schema != NULL);
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id);
generic_plugin_runtime_commit_ex_schema(runtime, schema, table_id, table_type,
valid_column);
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return 0;
}
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, const char *key, size_t key_len)
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
const char *key, size_t key_len)
{
struct maat_runtime *maat_rt = maat_instance->maat_rt;
if (NULL == maat_rt) {
@@ -569,11 +590,6 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, co
return ex_data_runtime_get_ex_data(ex_data_rt, key, key_len);
}
static void scan_count_inc(struct maat_state *mid)
{
mid->scan_cnt++;
}
struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id)
{
struct maat_state *outer_state = NULL;
@@ -587,7 +603,8 @@ struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id)
return outer_state;
}
struct maat_state *grab_state(struct maat_state **state, struct maat *maat_instance, int thread_id)
struct maat_state *grab_state(struct maat_state **state, struct maat *maat_instance,
int thread_id)
{
struct maat_state *mid = *state;
@@ -596,46 +613,51 @@ struct maat_state *grab_state(struct maat_state **state, struct maat *maat_insta
*state = mid;
//Maat_set_scan_status calls grap_mid() with thread_num=-1.
if (mid->thread_id >= 0) {
alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1);
alignment_int64_array_add(maat_instance->outer_state_cnt, thread_id, 1);
}
}
if (mid->thread_id < 0 && thread_id >= 0) {
mid->thread_id = thread_id;
alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1);
alignment_int64_array_add(maat_instance->outer_state_cnt, thread_id, 1);
}
if (NULL == mid->compile_mid) {
mid->compile_mid = maat_compile_state_new(thread_id);
alignment_int64_array_add(maat_instance->compile_mid_cnt, thread_id, 1);
if (NULL == mid->compile_state) {
mid->compile_state = maat_compile_state_new(thread_id);
alignment_int64_array_add(maat_instance->compile_state_cnt, thread_id, 1);
}
return mid;
}
int hit_group_to_compile(void *g2g_runtime, int *group_ids, int group_hit_cnt, int vt_id,
void *compile_runtime, int *compile_ids, size_t n_compile_id,
inline int scan_status_should_compile_NOT(struct maat_state *state)
{
if (state && (LAST_SCAN_SET == state->is_last_scan) && state->compile_state
&& maat_compile_state_has_NOT_clause(state->compile_state)) {
return 1;
}
return 0;
}
int hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t compile_ids_size,
size_t *n_hit_compile_id, struct maat_state *mid)
{
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int n_top_group_ids = group2group_runtime_get_top_groups(g2g_runtime, group_ids, group_hit_cnt, top_group_ids);
size_t n_all_group_ids = 0;
if (n_top_group_ids > 0) {
n_all_group_ids = group_hit_cnt + n_top_group_ids;
int compile_id_cnt = compile_runtime_match((struct compile_runtime *)compile_runtime,
compile_ids, compile_ids_size, mid);
*n_hit_compile_id = compile_id_cnt;
if (compile_id_cnt > 0) {
return MAAT_SCAN_HIT;
} else {
n_all_group_ids = group_hit_cnt;
return MAAT_SCAN_HALF_HIT;
}
}
int all_group_ids[n_all_group_ids] = {-1};
// maat state find compile_table_id, if not found, maat_instance->default_compile_table_id
int n_compile_ids = compile_runtime_match((struct compile_runtime *)compile_runtime, all_group_ids, n_all_group_ids,
vt_id, compile_ids, n_compile_id, mid);
*n_hit_compile_id = n_compile_ids;
if (n_compile_ids > 0) {
return MAAT_HIT;
} else {
return MAAT_HALF_HIT;
}
int maat_scan_flag(struct maat *instance, int table_id, int thread_id,
uint64_t flag, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state)
{
return 0;
}
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
@@ -651,18 +673,20 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
{
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) {
return -1;
return MAAT_SCAN_ERR;
}
struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, thread_id);
scan_count_inc(mid);
mid->scan_cnt++;
int virtual_table_id = 0;
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
int vtable_id = 0;
// struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
// SCAN_TYPE_IP, &virtual_table_id);
// if (NULL == table_schema) {
// return MAAT_ERR;
// return MAAT_SCAN_ERR;
// }
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
@@ -670,74 +694,243 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
assert(table_type == TABLE_TYPE_IP_PLUS);
// int table_id = table_schema_get_table_id(real_table);
void *ip_plus_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
if (NULL == ip_rt) {
return MAAT_SCAN_ERR;
}
// size_t rule_num = table_runtime_rule_count(table_rt);
// if (0 == rule_num) {
// return 0;
// }
int group_hit_cnt = ip_plus_runtime_scan_ipv4((struct ip_plus_runtime *)ip_plus_rt, thread_id, ip_addr,
group_ids, MAX_SCANNER_HIT_GROUP_NUM, virtual_table_id, mid);
int group_hit_cnt = ip_runtime_scan_ip((struct ip_runtime *)ip_rt, thread_id, IPv4,
(uint8_t *)&ip_addr, group_ids, sizeof(group_ids),
vtable_id, mid);
if (group_hit_cnt < 0) {
return MAAT_ERR;
} else if (0 == group_hit_cnt) {
return MAAT_OK;
} else {
// come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
void *compile_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id);
int ret = hit_group_to_compile(g2g_runtime, group_ids, group_hit_cnt, virtual_table_id, compile_runtime,
results, n_result, n_hit_result, mid);
return ret;
return MAAT_SCAN_ERR;
}
int compile_ret = 0;
if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
if (group_hit_cnt > 0) {
ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
}
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;
} else {
compile_table_id = mid->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid);
assert(mid->is_last_scan < LAST_SCAN_FINISHED);
if (LAST_SCAN_SET == mid->is_last_scan) {
mid->is_last_scan = LAST_SCAN_FINISHED;
}
}
if (compile_ret > 0) {
alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
}
if (0 == group_hit_cnt && compile_ret > 0) {
// hit NOT group
alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
}
maat_runtime_ref_dec(maat_instance->maat_rt, thread_id);
if (0 == compile_ret && group_hit_cnt > 0) {
return MAAT_SCAN_HALF_HIT;
}
return MAAT_SCAN_HIT;
}
int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id,
int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id,
uint8_t *ip_addr, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state)
{
return 0;
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == ip_addr) || (NULL == results) || (0 == n_result) ||
(NULL == state)) {
return MAAT_SCAN_ERR;
}
struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, thread_id);
mid->scan_cnt++;
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
int vtable_id = 0;
// struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
// SCAN_TYPE_IP, &virtual_table_id);
// if (NULL == table_schema) {
// return MAAT_SCAN_ERR;
// }
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
assert(table_type == TABLE_TYPE_IP_PLUS);
// int table_id = table_schema_get_table_id(real_table);
void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
if (NULL == ip_rt) {
return MAAT_SCAN_ERR;
}
// size_t rule_num = table_runtime_rule_count(table_rt);
// if (0 == rule_num) {
// return 0;
// }
int group_hit_cnt = ip_runtime_scan_ip((struct ip_runtime *)ip_rt, thread_id, IPv6, ip_addr,
group_ids, sizeof(group_ids), vtable_id, mid);
if (group_hit_cnt < 0) {
return MAAT_SCAN_ERR;
}
int compile_ret = 0;
if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
if (group_hit_cnt > 0) {
ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
}
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;
} else {
compile_table_id = mid->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid);
assert(mid->is_last_scan < LAST_SCAN_FINISHED);
if (LAST_SCAN_SET == mid->is_last_scan) {
mid->is_last_scan = LAST_SCAN_FINISHED;
}
}
if (compile_ret > 0) {
alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
}
if (0 == group_hit_cnt && compile_ret > 0) {
// hit NOT group
alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
}
maat_runtime_ref_dec(maat_instance->maat_rt, thread_id);
if (0 == compile_ret && group_hit_cnt > 0) {
return MAAT_SCAN_HALF_HIT;
}
return MAAT_SCAN_HIT;
}
int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
const char *data, size_t data_len, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state)
{
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) ||
(0 == n_result) || (NULL == state)) {
return MAAT_ERR;
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM)
|| (thread_id < 0) || (NULL == data) || (0 == data_len)
|| (NULL == results) || (0 == n_result) || (NULL == state)) {
return MAAT_SCAN_ERR;
}
struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, thread_id);
scan_count_inc(mid);
mid->scan_cnt++;
int virtual_table_id = 0;
int vtable_id = 0;
//TODO: by luis get virtual_table_id
// struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
// SCAN_TYPE_STRING, &virtual_table_id);
// if (NULL == table_schema) {
// return MAAT_ERR;
// return MAAT_SCAN_ERR;
// }
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)runtime, thread_id, data, data_len,
group_ids, MAX_SCANNER_HIT_GROUP_NUM, virtual_table_id, mid);
if (group_hit_cnt < 0) {
return MAAT_ERR;
} else if (0 == group_hit_cnt) {
return MAAT_OK;
} else {
// come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
void *compile_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id);
int ret = hit_group_to_compile(g2g_runtime, group_ids, group_hit_cnt, virtual_table_id, compile_runtime,
results, n_result, n_hit_result, mid);
return ret;
if (NULL == maat_instance->maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"maat_scan_string error because of maat_runtime is NULL");
return MAAT_SCAN_OK;
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
//TODO: is TABLE_TYPE_EXPR_PLUS
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
maat_instance->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
void *expr_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
if (NULL == expr_rt) {
return MAAT_SCAN_ERR;
}
int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, thread_id,
data, data_len, group_ids, sizeof(group_ids),
vtable_id, mid);
if (group_hit_cnt < 0) {
return MAAT_SCAN_ERR;
}
int compile_ret = 0;
if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
}
if (group_hit_cnt > 0 && table_type == TABLE_TYPE_EXPR_PLUS) {
}
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;
} else {
compile_table_id = mid->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid);
assert(mid->is_last_scan < LAST_SCAN_FINISHED);
if (LAST_SCAN_SET == mid->is_last_scan) {
mid->is_last_scan = LAST_SCAN_FINISHED;
}
}
if (compile_ret > 0) {
alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
}
if (0 == group_hit_cnt && compile_ret > 0) {
// hit NOT group
alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
}
maat_runtime_ref_dec(maat_instance->maat_rt, thread_id);
if (0 == compile_ret && group_hit_cnt > 0) {
return MAAT_SCAN_HALF_HIT;
}
return MAAT_SCAN_HIT;
}
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id)
@@ -745,8 +938,9 @@ struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, i
return NULL;
}
int maat_scan_stream(struct maat_stream **stream, int thread_id, const char *data, int data_len,
int results[], size_t *n_result, struct maat_state **state)
int maat_scan_stream(struct maat_stream **stream, int thread_id,
const char *data, int data_len, int results[],
size_t *n_result, struct maat_state **state)
{
return 0;
}
@@ -756,19 +950,27 @@ void maat_scan_stream_close(struct maat_stream **stream)
}
int maat_state_set_scan_district(struct maat *maat_instance, struct maat_state **state, const char *district, size_t district_len)
int maat_state_set_scan_district(struct maat *maat_instance,
struct maat_state **state,
const char *district,
size_t district_len)
{
if (NULL == maat_instance->maat_rt || NULL == district || district_len <= 0) {
if (NULL == maat_instance->maat_rt || NULL == district ||
district_len <= 0) {
return -1;
}
struct maat_state *mid = grab_state(state, maat_instance, -1);
int map_ret = maat_kv_read_unNull(maat_instance->maat_rt->district_map, district, district_len, &(mid->district_id));
if (map_ret < 0) {
int ret = table_manager_set_scan_district(maat_instance->tbl_mgr,
district, district_len,
&(mid->district_id));
// int map_ret = maat_kv_read_unNull(maat_instance->maat_rt->district_map,
// district, district_len, &(mid->district_id));
if (ret < 0) {
mid->district_id = DISTRICT_UNKNOWN;
}
mid->is_set_district = 1;
mid->is_set_district = DISTRICT_FLAG_SET;
return 0;
}
@@ -780,8 +982,8 @@ int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **sta
}
struct maat_state *mid = grab_state(state, maat_instance, -1);
assert(mid->is_last_scan == 0);
mid->is_last_scan = 1;
assert(mid->is_last_scan == LAST_SCAN_UNSET);
mid->is_last_scan = LAST_SCAN_SET;
return 0;
}
@@ -799,26 +1001,27 @@ int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_st
return 0;
}
size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid,
size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *state,
struct maat_hit_path *paths, size_t n_path)
{
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
if (state->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;
} else {
compile_table_id = mid->compile_table_id;
compile_table_id = state->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt,
(struct group2group_runtime *)g2g_runtime, mid->compile_mid, paths, n_path);
return 0;
assert(NULL != compile_rt && NULL != g2g_runtime);
return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt,
(struct group2group_runtime *)g2g_runtime,
state->compile_state, paths, n_path);
}
size_t maat_get_hit_objects(struct maat_compile_state *compile_state, struct maat_hit_object *objs, size_t n_objs)
size_t maat_get_hit_objects(struct maat_compile_state *compile_state,
struct maat_hit_object *objs, size_t n_objs)
{
return 0;
}
@@ -829,7 +1032,7 @@ int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **sta
struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, 0);
if (NULL == mid->compile_mid || NULL == maat_instance->maat_rt) {
if (NULL == mid->compile_state || NULL == maat_instance->maat_rt) {
return 0;
}
@@ -853,13 +1056,13 @@ void maat_state_free(struct maat_state **state)
mid = *state;
if (mid->thread_id >= 0) {
alignment_int64_array_add(mid->maat_instance->outer_mid_cnt, mid->thread_id, -1);
alignment_int64_array_add(mid->maat_instance->outer_state_cnt, mid->thread_id, -1);
}
if (mid->compile_mid != NULL) {
maat_compile_state_free(mid->compile_mid);
mid->compile_mid = NULL;
alignment_int64_array_add(mid->maat_instance->compile_mid_cnt, mid->thread_id, -1);
if (mid->compile_state != NULL) {
maat_compile_state_free(mid->compile_state);
mid->compile_state = NULL;
alignment_int64_array_add(mid->maat_instance->compile_state_cnt, mid->thread_id, -1);
}
mid->maat_instance = NULL;
@@ -870,4 +1073,4 @@ void maat_state_free(struct maat_state **state)
int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj)
{
return 0;
}
}

31
src/maat_command.cpp
View File

@@ -13,7 +13,6 @@
#include <unistd.h>
#include <string.h>
#include "utils.h"
#include "maat_utils.h"
#include "maat_command.h"
#include "maat_rule.h"
@@ -52,7 +51,8 @@ redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...
return (redisReply *)reply;
}
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db, struct log_handle *logger)
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger)
{
struct timeval connect_timeout;
connect_timeout.tv_sec = 0;
@@ -116,10 +116,12 @@ void maat_cmd_clear_rule_cache(struct serial_rule *s_rule)
memset(s_rule, 0, sizeof(struct serial_rule));
}
int connect_redis_for_write(struct source_redis_ctx *mr_ctx, struct log_handle *logger)
int connect_redis_for_write(struct source_redis_ctx *mr_ctx,
struct log_handle *logger)
{
assert(mr_ctx->write_ctx == NULL);
mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, logger);
mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port,
mr_ctx->redis_db, logger);
if (NULL == mr_ctx->write_ctx) {
return -1;
} else {
@@ -130,7 +132,8 @@ int connect_redis_for_write(struct source_redis_ctx *mr_ctx, struct log_handle *
redisContext *get_redis_ctx_for_write(struct maat *maat_instance)
{
if (NULL == maat_instance->mr_ctx.write_ctx) {
int ret = connect_redis_for_write(&(maat_instance->mr_ctx), maat_instance->logger);
int ret = connect_redis_for_write(&(maat_instance->mr_ctx),
maat_instance->logger);
if(ret!=0)
{
return NULL;
@@ -139,8 +142,9 @@ redisContext *get_redis_ctx_for_write(struct maat *maat_instance)
return maat_instance->mr_ctx.write_ctx;
}
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id,
const char *table_name, const char *line, long long timeout)
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
unsigned long rule_id, const char *table_name,
const char *line, long long timeout)
{
memset(rule, 0, sizeof(struct serial_rule));
rule->op = op;
@@ -153,7 +157,8 @@ void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
}
}
int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq)
int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
int valid_column_seq)
{
int column_seq = 0;
@@ -204,7 +209,8 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
size_t len = 0;
int ret = get_column_pos(line, column_seq, &offset, &len);
// 0 is also a valid value for some non-MAAT producer.
if (ret < 0 || offset >= strlen(line) || (line[offset] != '1' && line[offset] != '0')) {
if (ret < 0 || offset >= strlen(line) || (line[offset] != '1' &&
line[offset] != '0')) {
return -1;
}
@@ -306,7 +312,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, line_rule->table_name);
if (table_id < 0) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND, "Command set line id %d failed: unknown table %s",
log_error(maat_instance->logger, MODULE_MAAT_COMMAND,
"Command set line id %d failed: unknown table %s",
line_rule->rule_id, line_rule->table_name);
FREE(s_rule);
return -1;
@@ -328,8 +335,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
absolute_expire_time = server_time + line_rule->expire_after;
}
maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name,
line_rule->table_line, absolute_expire_time);
maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id,
line_rule->table_name, line_rule->table_line, absolute_expire_time);
int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time, maat_instance->logger);
if (success_cnt != 1) {

481
src/maat_compile.cpp
View File

File diff suppressed because it is too large Load Diff

44
src/maat_config_monitor.cpp
View File

@@ -16,7 +16,6 @@
#include <dirent.h>
#include <unistd.h>
#include "utils.h"
#include "maat_utils.h"
#include "maat_rule.h"
#include "json2iris.h"
@@ -84,7 +83,8 @@ const char *path2filename(const char *path)
return path + i + 1;
}
char *read_nxt_line_from_buff(const char *buff, size_t buff_size, size_t *offset, char *line, int line_size)
char *read_nxt_line_from_buff(const char *buff, size_t buff_size,
size_t *offset, char *line, int line_size)
{
int this_offset=0;
const char* p;
@@ -221,7 +221,7 @@ int get_new_idx_path(long long current_version, const char *file_dir,
char ***idx_path, size_t *idx_num, struct log_handle *logger)
{
struct dirent **namelist = NULL;
int update_type = CM_UPDATE_TYPE_NONE;
int update_type = MAAT_UPDATE_TYPE_NONE;
int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort);
if (n < 0) {
@@ -285,7 +285,7 @@ int get_new_idx_path(long long current_version, const char *file_dir,
(*idx_path)[0] = (char *)malloc(path_len);
snprintf((*idx_path)[0], path_len, "%s/%s", file_dir, namelist[full_file_idx]->d_name);
*idx_num = 1;
update_type = CM_UPDATE_TYPE_FULL;
update_type = MAAT_UPDATE_TYPE_FULL;
} else if (latest_inc_version > current_version) {
//inc update,it's possible that do inc after full update in this function,but we'll process it at next loop.
*idx_path = (char **)malloc(sizeof(char **) * inc_idx_num);
@@ -295,9 +295,9 @@ int get_new_idx_path(long long current_version, const char *file_dir,
snprintf((*idx_path)[i], path_len, "%s/%s", file_dir, namelist[inc_file_idx[i]]->d_name);
}
*idx_num = inc_idx_num;
update_type = CM_UPDATE_TYPE_INC;
update_type = MAAT_UPDATE_TYPE_INC;
} else {
update_type = CM_UPDATE_TYPE_NONE;
update_type = MAAT_UPDATE_TYPE_NONE;
}
FREE(inc_file_idx);
@@ -324,7 +324,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
memset(table_array, 0, sizeof(table_array));
int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num, logger);
if (update_type != CM_UPDATE_TYPE_NONE) {
if (update_type != MAAT_UPDATE_TYPE_NONE) {
for (i = 0; i < idx_path_num; i++) {
log_info(logger, MODULE_CONFIG_MONITOR, "load %s", idx_path_array[i]);
int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM);
@@ -370,12 +370,15 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
size_t uncompressed_buff_sz = 0;
log_info(maat_instance->logger, MODULE_CONFIG_MONITOR,
"Maat initial with JSON file %s, formating...", json_filename);
"Maat initial with JSON file %s, formating...",
json_filename);
if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) {
ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo,
(unsigned char **)&decrypted_buff, &decrypted_buff_sz,
err_str, err_str_sz);
ret = decrypt_open(json_filename, maat_instance->decrypt_key,
maat_instance->decrypt_algo,
(unsigned char **)&decrypted_buff,
&decrypted_buff_sz,
err_str, err_str_sz);
if (ret < 0) {
log_error(maat_instance->logger, MODULE_CONFIG_MONITOR,
"Decrypt Maat JSON file %s failed", json_filename);
@@ -387,7 +390,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
}
if (maat_instance->maat_json_is_gzipped) {
ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz);
ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff,
&uncompressed_buff_sz);
FREE(json_buff);
if (ret < 0) {
log_error(maat_instance->logger, MODULE_CONFIG_MONITOR,
@@ -410,7 +414,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
}
ret = json2iris((const char*)json_buff, json_filename, NULL,
maat_instance->json_ctx.iris_file, sizeof(maat_instance->json_ctx.iris_file),
maat_instance->json_ctx.iris_file,
sizeof(maat_instance->json_ctx.iris_file),
strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL,
strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL,
maat_instance->logger);
@@ -422,17 +427,20 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
}
if (!maat_instance->is_running) {
strncpy(maat_instance->json_ctx.json_file, json_filename, sizeof(maat_instance->json_ctx.json_file));
strncpy(maat_instance->json_ctx.json_file, json_filename,
sizeof(maat_instance->json_ctx.json_file));
}
ret = stat(json_filename, &fstat_buf);
maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim;
md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5);
log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, "JSON file %s md5: %s, generate index file %s OK",
maat_instance->json_ctx.json_file,
maat_instance->json_ctx.effective_json_md5,
maat_instance->json_ctx.iris_file);
log_info(maat_instance->logger, MODULE_CONFIG_MONITOR,
"JSON file %s md5: %s, generate index file %s OK",
maat_instance->json_ctx.json_file,
maat_instance->json_ctx.effective_json_md5,
maat_instance->json_ctx.iris_file);
maat_instance->input_mode = DATA_SOURCE_JSON_FILE;
return 0;

70
src/maat_ex_data.cpp
View File

@@ -15,7 +15,6 @@
#include "uthash/utarray.h"
#include "log/log.h"
#include "rcu_hash.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_ex_data.h"
@@ -48,8 +47,9 @@ void cache_row_free(void *p)
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn,
struct log_handle *logger)
struct ex_data_runtime *
ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn,
struct log_handle *logger)
{
struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1);
@@ -149,26 +149,31 @@ void ex_data_schema_free(struct ex_data_schema *ex_schema)
FREE(ex_schema);
}
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema)
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt,
struct ex_data_schema *schema)
{
ex_data_rt->ex_schema = schema;
}
void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, struct ex_container_ctx *container_ctx)
void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt,
struct ex_container_ctx *container_ctx)
{
rcu_hash_set_user_ctx(ex_data_rt->htable, container_ctx);
}
struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt)
struct ex_container_ctx *
ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt)
{
return (struct ex_container_ctx *)rcu_hash_get_user_ctx(ex_data_rt->htable);
}
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len)
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row,
const char *key, size_t key_len)
{
void *ex_data = NULL;
struct ex_data_schema *ex_schema = ex_data_rt->ex_schema;
ex_schema->new_func(ex_data_rt->table_id, key, row, &ex_data, ex_schema->argl, ex_schema->argp);
ex_schema->new_func(ex_data_rt->table_id, key, row, &ex_data,
ex_schema->argl, ex_schema->argp);
return ex_data;
}
@@ -194,11 +199,14 @@ void ex_data_container_free(void *ctx, void *data)
void *argp = container_ctx->ex_schema->argp;
struct ex_data_container *ex_container = (struct ex_data_container *)data;
if (ex_container->ex_data != NULL && container_ctx->ex_schema->free_func != NULL) {
container_ctx->ex_schema->free_func(container_ctx->table_id, &(ex_container->ex_data), argl, argp);
if (ex_container->ex_data != NULL
&& container_ctx->ex_schema->free_func != NULL) {
container_ctx->ex_schema->free_func(container_ctx->table_id,
&(ex_container->ex_data), argl, argp);
}
if (ex_container->custom_data != NULL && container_ctx->custom_data_free != NULL) {
if (ex_container->custom_data != NULL
&& container_ctx->custom_data_free != NULL) {
container_ctx->custom_data_free(ex_container->custom_data);
}
@@ -211,10 +219,12 @@ int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
{
struct ex_data_container *tmp_container = NULL;
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (tmp_container != NULL) {
log_error(ex_data_rt->logger, MODULE_EX_DATA,
"ex_data_runtime add ex container error: already exist same key:%s", key);
"ex_data_runtime add ex container error: already exist same key:%s",
key);
return -1;
}
@@ -227,10 +237,12 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{
struct ex_data_container *tmp_container = NULL;
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (NULL == tmp_container) {
log_error(ex_data_rt->logger, MODULE_EX_DATA,
"ex_data_runtime del ex container error: no such key:%s", key);
"ex_data_runtime del ex container error: no such key:%s",
key);
return -1;
}
@@ -239,22 +251,30 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
return 0;
}
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len)
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{
struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
struct ex_data_container *ex_container = NULL;
ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (NULL == ex_container) {
return NULL;
}
void *dup_ex_data = NULL;
ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, &(ex_container->ex_data),
ex_data_rt->ex_schema->argl, ex_data_rt->ex_schema->argp);
ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data,
&(ex_container->ex_data),
ex_data_rt->ex_schema->argl,
ex_data_rt->ex_schema->argp);
return dup_ex_data;
}
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len)
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{
struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
struct ex_data_container *ex_container = NULL;
ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (NULL == ex_container) {
return NULL;
}
@@ -267,12 +287,8 @@ size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt)
return rcu_hash_count(ex_data_rt->htable);
}
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container)
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container ***ex_container)
{
return rcu_hash_list_updating_data(ex_data_rt->htable, (void ***)ex_container);
}
int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt)
{
return rcu_hash_updating_flag(ex_data_rt->htable);
}

215
src/maat_expr.cpp
View File

@@ -11,7 +11,6 @@
#include <string.h>
#include <assert.h>
#include "utils.h"
#include "maat_expr.h"
#include "adapter_hs.h"
#include "maat_utils.h"
@@ -20,6 +19,8 @@
#include "rcu_hash.h"
#include "maat_rule.h"
#include "maat_compile.h"
#include "maat_group.h"
#include "alignment.h"
#include "maat_garbage_collection.h"
#define MAX_DISTRICT_STR 128
@@ -36,6 +37,7 @@ struct expr_schema {
int is_hexbin_column;
enum hs_scan_mode scan_mode; /* adapter_hs scan mode */
int table_id; //ugly
struct table_manager *ref_tbl_mgr;
};
enum expr_type {
@@ -56,7 +58,7 @@ enum match_method {
struct expr_item {
int item_id;
int group_id;
char district[MAX_DISTRICT_STR];
int district_id;
char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type;
enum match_method match_method;
@@ -80,9 +82,8 @@ struct expr_runtime {
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
// long long *scan_cnt;
// long long *hit_cnt;
// long long *not_grp_hit_cnt;
long long *scan_cnt;
long long *hit_cnt;
// long long *stream_num;
};
@@ -139,11 +140,13 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
int db_hexbin = -1;
int expr_type = -1;
int match_method_type = -1;
enum table_type table_type = TABLE_TYPE_INVALID;
struct expr_item *expr_item = ALLOC(struct expr_item, 1);
int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no item_id",
log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no item_id",
expr_schema->table_id, line);
goto error;
}
@@ -151,32 +154,40 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no group_id",
log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no group_id",
expr_schema->table_id, line);
goto error;
}
expr_item->group_id = atoi(line + column_offset);
table_type = table_manager_get_table_type(expr_schema->tbl_mgr, expr_schema->table_id);
//TODO
#if 0
if (table_item->table_type == TABLE_TYPE_EXPR_PLUS) {
#if 1
if (table_type == TABLE_TYPE_EXPR_PLUS) {
ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len);
if (ret < 0) {
return -1;
goto error;
}
if (column_len >= MAX_DISTRICT_STR) {
log_error(logger, MODULE_EXPR,
"update error: expr table[%s]:item_id[%d] district length too long",
table_name, expr_item->item_id);
return -1;
"expr table(table_id:%d) line:%s district length too long",
expr_schema->table_id, line);
goto error;
}
memcpy(expr_item->district, (line + column_offset), column_len);
char district[MAX_DISTRICT_STR] = {0};
memcpy(district, (line + column_offset), column_len);
assert(strlen(district) > 0);
str_unescape(district);
expr_item->district_id = get_district_id()
}
#endif
ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no keywords",
log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no keywords",
expr_schema->table_id, line);
goto error;
}
@@ -191,7 +202,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no expr_type",
log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no expr_type",
expr_schema->table_id, line);
goto error;
}
@@ -201,7 +213,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no match_method",
log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no match_method",
expr_schema->table_id, line);
goto error;
}
@@ -211,7 +224,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no is_hexbin",
log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no is_hexbin",
expr_schema->table_id, line);
goto error;
}
@@ -248,7 +262,8 @@ void expr_item_free(struct expr_item *expr_item)
FREE(expr_item);
}
void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
int read_cnt = 0;
struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1);
@@ -267,15 +282,18 @@ void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *lo
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_EXPR, "table %s has no custom column", table_name);
log_error(logger, MODULE_EXPR,
"table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "scan_mode");
if (custom_item != NULL && custom_item->type == cJSON_String) {
ret = maat_kv_read(scan_mode_map, custom_item->valuestring, (int*)&(expr_schema->scan_mode));
ret = maat_kv_read(scan_mode_map, custom_item->valuestring,
(int*)&(expr_schema->scan_mode));
if (ret < 0) {
log_error(logger, MODULE_EXPR, "scan_mode %s illegal", custom_item->valuestring);
log_error(logger, MODULE_EXPR, "scan_mode %s illegal",
custom_item->valuestring);
goto error;
}
read_cnt++;
@@ -323,6 +341,8 @@ void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *lo
read_cnt++;
}
expr_schema->tbl_mgr = tbl_mgr;
if (read_cnt < 8) {
goto error;
}
@@ -359,7 +379,8 @@ void expr_ex_data_free(void *user_ctx, void *data)
expr_rule_free(expr_rule);
}
void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
void *expr_runtime_new(void *expr_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == expr_schema) {
@@ -376,10 +397,8 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbag
expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger;
// expr_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
// expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
// expr_rt->not_grp_hit_cnt = alignment_int64_array_alloc(max_thread_num);
// expr_rt->stream_num = alignment_int64_array_alloc(max_thread_num);
expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
expr_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
return expr_rt;
}
@@ -411,11 +430,23 @@ void expr_runtime_free(void *expr_runtime)
HASH_DELETE(hh, expr_rt->item_hash, item);
maat_item_free(item, expr_rt->item_user_data_free);
}
if (expr_rt->hit_cnt != NULL) {
alignment_int64_array_free(expr_rt->hit_cnt);
expr_rt->hit_cnt = NULL;
}
if (expr_rt->scan_cnt != NULL) {
alignment_int64_array_free(expr_rt->scan_cnt);
expr_rt->scan_cnt = NULL;
}
FREE(expr_rt);
}
int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len,
and_expr_t *expr_rule, int is_valid, struct log_handle *logger)
int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
size_t key_len, and_expr_t *expr_rule,
int is_valid, struct log_handle *logger)
{
void *data = NULL;
@@ -424,7 +455,8 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_
data = rcu_hash_find(expr_rt->htable, key, key_len);
if (NULL == data) {
log_error(logger, MODULE_EXPR,
"the key of expr rule not exist, can't be deleted, expr_id:%d", expr_rule->expr_id);
"the key of expr rule not exist, can't be deleted, expr_id:%d",
expr_rule->expr_id);
return -1;
}
rcu_hash_del(expr_rt->htable, key, key_len);
@@ -433,7 +465,8 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_
data = rcu_hash_find(expr_rt->htable, key, key_len);
if (data != NULL) {
log_error(logger, MODULE_EXPR,
"the key of expr rule already exist, can't be added, expr_id:%d", expr_rule->expr_id);
"the key of expr rule already exist, can't be added, expr_id:%d",
expr_rule->expr_id);
return -1;
}
rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule);
@@ -461,7 +494,8 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type)
}
#define MAAT_MAX_EXPR_ITEM_NUM 8
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger)
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item,
struct log_handle *logger)
{
size_t i = 0;
size_t sub_expr_cnt = 0;
@@ -481,7 +515,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl
if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_EXPR,
"expr item_id:%d too many patterns", expr_item->item_id);
"expr item_id:%d too many patterns",
expr_item->item_id);
return NULL;
}
@@ -515,7 +550,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl
return expr_rule;
}
int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column)
int expr_runtime_update(void *expr_runtime, void *expr_schema,
const char *line, int valid_column)
{
if (NULL == expr_runtime || NULL == expr_schema) {
return -1;
@@ -553,7 +589,8 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line,
HASH_FIND_INT(expr_rt->item_hash, &item_id, item);
if (item) {
log_error(expr_rt->logger, MODULE_EXPR,
"expr runtime add item %d to item_hash failed, already exist", item_id);
"expr runtime add item %d to item_hash failed, already exist",
item_id);
return -1;
}
@@ -572,14 +609,16 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line,
expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger);
expr_item_free(expr_item);
if (NULL == expr_rule) {
log_error(expr_rt->logger, MODULE_EXPR, "transform expr table(table_id:%d) item to expr_rule failed, item_id:%d",
log_error(expr_rt->logger, MODULE_EXPR,
"transform expr table(table_id:%d) item to expr_rule failed, item_id:%d",
schema->table_id, item_id);
return -1;
}
}
char *key = (char *)&item_id;
ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, is_valid, expr_rt->logger);
ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule,
is_valid, expr_rt->logger);
if (ret < 0) {
if (expr_rule != NULL) {
expr_rule_free(expr_rule);
@@ -622,20 +661,25 @@ int expr_runtime_commit(void *expr_runtime)
struct adapter_hs *old_adapter_hs = NULL;
log_info(expr_rt->logger, MODULE_EXPR,
"committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt);
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger);
"committing %zu expr rules for rebuilding adapter_hs engine",
rule_cnt);
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode,
expr_rt->n_worker_thread,
rules, rule_cnt,
expr_rt->logger);
if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR,
"rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt);
"rebuild adapter_hs engine failed when update %zu expr rules",
rule_cnt);
ret = -1;
}
old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs;
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy);
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs,
(void (*)(void*))adapter_hs_destroy);
rcu_hash_commit(expr_rt->htable);
expr_rt->rule_num = rcu_hash_count(expr_rt->htable);
rule_cnt = rcu_hash_updating_count(expr_rt->htable);
assert(rule_cnt == 0);
@@ -645,71 +689,37 @@ int expr_runtime_commit(void *expr_runtime)
return ret;
}
int expr_runtime_updating_flag(void *expr_runtime)
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
return rcu_hash_updating_flag(expr_rt->htable);
}
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len,
int group_id_array[], size_t n_group_id_array, int virtual_table_id, struct maat_state *state)
{
if (NULL == expr_rt) {
return -1;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
size_t n_hit_item = 0;
int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, hit_item_ids, &n_hit_item);
if (ret < 0) {
return -1;
}
if (0 == n_hit_item) {
return 0;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(expr_rt->logger, MODULE_EXPR,
"hit expr item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_compile_state *compile_state = state->compile_mid;
//tranform item_id to group_id
struct maat_item *item = NULL;
size_t n_group_id = 0;
size_t i = 0;
for (i = 0; i < n_hit_item; i++) {
HASH_FIND_INT(expr_rt->item_hash, &(hit_item_ids[i]), item);
assert(item != NULL);
if (!item) {
// should not come here
continue;
}
if (n_group_id >= n_group_id_array) {
n_group_id = n_group_id_array;
//Prevent group_id_array out of bounds
} else {
group_id_array[n_group_id++] = item->group_id;
}
maat_compile_state_update_hit_path(compile_state, hit_item_ids[i], item->group_id, virtual_table_id, state->scan_cnt, i);
}
// literal_id{group_id,vt_id} to clause_id
// STEP 1: get compile table runtime
int compile_table_id = -1;
if (state->compile_table_id == -1) {
compile_table_id = state->maat_instance->default_compile_table_id;
} else {
compile_table_id = state->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id);
enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, compile_table_id);
assert(table_type == TABLE_TYPE_COMPILE);
// STEP 2: get the specified compile table's hit clause_id array by literal_id
for (i = 0; i < n_group_id; i++) {
maat_compile_state_update_hit_clause(compile_state, compile_rt, group_id_array[i], virtual_table_id);
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, n_hit_item,
group_ids, group_ids_size, &group_hit_cnt, state);
if (ret < 0) {
return -1;
}
return n_group_id;
return group_hit_cnt;
}
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
@@ -722,14 +732,15 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
expr_rt->hs_stream = hs_stream;
}
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len,
int result[], size_t *n_result)
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data,
size_t data_len, int result[], size_t *n_result)
{
if (NULL == expr_rt) {
return -1;
}
return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, result, n_result);
return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len,
result, n_result);
}
void expr_runtime_stream_close(struct expr_runtime *expr_rt)
@@ -739,3 +750,13 @@ void expr_runtime_stream_close(struct expr_runtime *expr_rt)
expr_rt->hs_stream = NULL;
}
}
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id)
{
alignment_int64_array_add(expr_rt->hit_cnt, thread_id, 1);
}
long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread)
{
return alignment_int64_array_sum(expr_rt->hit_cnt, n_thread);
}

7
src/maat_garbage_collection.cpp
View File

@@ -8,13 +8,13 @@
***********************************************************************************************
*/
#include "utils.h"
#include <time.h>
#include <stdlib.h>
#include <sys/queue.h>
#include <assert.h>
#include "maat_utils.h"
struct maat_garbage_bag {
time_t create_time;
int timeout;
@@ -60,7 +60,8 @@ size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin)
return bin->bag_cnt;
}
void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void (* func)(void *))
void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage,
void (* func)(void *))
{
struct maat_garbage_bag *bag = ALLOC(struct maat_garbage_bag, 1);

149
src/maat_group.cpp
View File

@@ -12,7 +12,6 @@
#include <pthread.h>
#include "maat_group.h"
#include "utils.h"
#include "maat_utils.h"
#include "uthash/uthash.h"
#include "igraph/igraph.h"
@@ -22,23 +21,28 @@
struct group2group_item {
int group_id;
int superior_group_id;
int super_group_id;
};
struct group2group_schema {
int group_id_column;
int superior_group_id_column;
int super_group_id_column;
int table_id;//ugly
struct table_manager *tbl_mgr;
};
struct group2group_runtime {
struct maat_group_topology *group_topo;
struct maat_group {
igraph_integer_t vertex_id;
int group_id;
int ref_by_compile_cnt;
int ref_by_super_group_cnt;
int ref_by_sub_group_cnt;
uint32_t rule_num;
uint32_t updating_rule_num;
size_t top_group_cnt;
int *top_group_ids;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
UT_hash_handle hh_group_id;
UT_hash_handle hh_vertex_id;
};
struct maat_group_topology {
@@ -53,7 +57,18 @@ struct maat_group_topology {
struct log_handle *logger;
};
void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
struct group2group_runtime {
struct maat_group_topology *group_topo;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
};
void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
int read_cnt = 0;
struct group2group_schema *g2g_schema = ALLOC(struct group2group_schema, 1);
@@ -67,7 +82,8 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_GROUP, "table %s has no custom column", table_name);
log_error(logger, MODULE_GROUP,
"table %s has no custom column", table_name);
goto error;
}
@@ -77,12 +93,14 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "superior_group_id");
custom_item = cJSON_GetObjectItem(item, "super_group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2g_schema->superior_group_id_column = custom_item->valueint;
g2g_schema->super_group_id_column = custom_item->valueint;
read_cnt++;
}
g2g_schema->tbl_mgr = tbl_mgr;
if (read_cnt < 3) {
goto error;
}
@@ -117,7 +135,8 @@ struct maat_group_topology *maat_group_topology_new(struct log_handle *logger)
return group_topo;
}
void *group2group_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
void *group2group_runtime_new(void *g2g_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
struct group2group_runtime *g2g_rt = ALLOC(struct group2group_runtime, 1);
@@ -167,6 +186,16 @@ void group2group_runtime_free(void *g2g_runtime)
FREE(g2g_rt);
}
void maat_group_ref_inc(struct maat_group *group)
{
group->ref_by_compile_cnt++;
}
void maat_group_ref_dec(struct maat_group *group)
{
group->ref_by_compile_cnt--;
}
struct group2group_item *
group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
struct log_handle *logger)
@@ -184,14 +213,15 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
}
g2g_item->group_id = atoi(line + column_offset);
ret = get_column_pos(line, g2g_schema->superior_group_id_column, &column_offset, &column_len);
ret = get_column_pos(line, g2g_schema->super_group_id_column,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_GROUP,
"group2group table(table_id:%d) line:%s has no superior_group_id",
"group2group table(table_id:%d) line:%s has no super_group_id",
g2g_schema->table_id, line);
goto error;
}
g2g_item->superior_group_id = atoi(line + column_offset);
g2g_item->super_group_id = atoi(line + column_offset);
return g2g_item;
error:
@@ -251,13 +281,14 @@ void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *grou
struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL);
assert(group->ref_by_compile_cnt == 0 && group->ref_by_superior_group_cnt == 0);
assert(group->ref_by_compile_cnt == 0 && group->ref_by_super_group_cnt == 0);
igraph_vector_init(&v, 8);
igraph_neighbors(&group_topo->group_graph, &v, group->vertex_id, IGRAPH_ALL);
if (igraph_vector_size(&v) > 0) {
print_igraph_vector(&v, buff, sizeof(buff));
log_error(group_topo->logger, MODULE_GROUP, "Del group %d exception, still reached by %s.",
group->vertex_id, buff);
log_error(group_topo->logger, MODULE_GROUP,
"Del group %d exception, still reached by %s.",
group->vertex_id, buff);
assert(0);
}
igraph_vector_destroy(&v);
@@ -288,7 +319,8 @@ struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_i
return group;
}
int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int superior_group_id)
int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id,
int super_group_id)
{
if (NULL == g2g_runtime) {
return -1;
@@ -305,30 +337,34 @@ int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int
group = group2group_runtime_add_group(g2g_runtime, group_id);
}
struct maat_group *superior_group = group2group_runtime_find_group(g2g_runtime, superior_group_id);
if (NULL == superior_group) {
superior_group = group2group_runtime_add_group(g2g_runtime, superior_group_id);
struct maat_group *super_group = group2group_runtime_find_group(g2g_runtime,
super_group_id);
if (NULL == super_group) {
super_group = group2group_runtime_add_group(g2g_runtime, super_group_id);
}
ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id,
super_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
//No duplicated edges between two groups.
if (edge_id > 0) {
log_error(g2g_rt->logger, MODULE_GROUP,
"Add group %d to group %d failed, relation already exisited.",
group->group_id, superior_group->group_id);
group->group_id, super_group->group_id);
ret = -1;
} else {
igraph_add_edge(&group_topo->group_graph, group->vertex_id, superior_group->vertex_id);
group->ref_by_superior_group_cnt++;
superior_group->ref_by_subordinate_group_cnt++;
igraph_add_edge(&group_topo->group_graph, group->vertex_id,
super_group->vertex_id);
group->ref_by_super_group_cnt++;
super_group->ref_by_sub_group_cnt++;
ret = 0;
}
return ret;
}
int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, int superior_group_id)
int group2group_runtime_remove_group_from_group(void *g2g_runtime,
int group_id, int super_group_id)
{
if (NULL == g2g_runtime) {
return -1;
@@ -341,15 +377,16 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id,
if (NULL == group) {
log_error(g2g_rt->logger, MODULE_GROUP,
"Del group %d from group %d failed, group %d not exisited.",
group_id, superior_group_id, group_id);
group_id, super_group_id, group_id);
return -1;
}
struct maat_group *superior_group = group2group_runtime_find_group(g2g_runtime, superior_group_id);
if (NULL == superior_group) {
struct maat_group *super_group = group2group_runtime_find_group(g2g_runtime,
super_group_id);
if (NULL == super_group) {
log_error(g2g_rt->logger, MODULE_GROUP,
"Del group %d from group %d failed, superior group %d not exisited.",
group_id, superior_group_id, superior_group_id);
group_id, super_group_id, super_group_id);
return -1;
}
@@ -364,7 +401,8 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id,
//first edge, the fifth is the first vertex of the second edge and so on. The last element
//of the argument list must be -1 to denote the end of the argument list.
//https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small
int ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1);
int ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id,
super_group->vertex_id, -1);
assert(ret==IGRAPH_SUCCESS);
// ignore no such edge to abort().
igraph_set_error_handler(igraph_error_handler_ignore);
@@ -377,8 +415,8 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id,
return -1;
}
group->ref_by_superior_group_cnt--;
superior_group->ref_by_subordinate_group_cnt--;
group->ref_by_super_group_cnt--;
super_group->ref_by_sub_group_cnt--;
return 0;
}
@@ -405,7 +443,7 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
}
struct maat_group *group = NULL, *tmp = NULL;
struct maat_group *superior_group = NULL;
struct maat_group *super_group = NULL;
int tmp_vid=0;
size_t top_group_cnt=0;
int* temp_group_ids=NULL;
@@ -426,10 +464,11 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
HASH_ITER (hh_group_id, group_topo->hash_group_by_id, group, tmp) {
top_group_cnt = 0;
temp_group_ids = NULL;
//Orphan, Not reference by any one, free it.
if (0 == group->ref_by_compile_cnt
&& 0 == group->ref_by_superior_group_cnt
&& 0 == group->ref_by_subordinate_group_cnt) {
&& 0 == group->ref_by_super_group_cnt
&& 0 == group->ref_by_sub_group_cnt) {
FREE(group->top_group_ids);
group2group_runtime_remove_group(g2g_runtime, group);
@@ -437,8 +476,8 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
}
//A group is need to build top groups when it has items and referenced by superior groups or compiles.
if (group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0) {
if (0 == group->ref_by_superior_group_cnt) {
if (group->ref_by_compile_cnt > 0 || group->ref_by_super_group_cnt > 0) {
if (0 == group->ref_by_super_group_cnt) {
//fast path, group is only referenced by compile rules.
top_group_cnt = 1;
temp_group_ids = ALLOC(int, top_group_cnt);
@@ -456,11 +495,12 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
break;
}
HASH_FIND(hh_vertex_id, group_topo->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group);
HASH_FIND(hh_vertex_id, group_topo->hash_group_by_vertex, &tmp_vid,
sizeof(tmp_vid), super_group);
//including itself
if (superior_group->ref_by_compile_cnt > 0) {
temp_group_ids[top_group_cnt] = superior_group->group_id;
if (super_group->ref_by_compile_cnt > 0) {
temp_group_ids[top_group_cnt] = super_group->group_id;
top_group_cnt++;
}
}
@@ -479,10 +519,11 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
return 0;
}
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line,
int valid_column)
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
const char *line, int valid_column)
{
if (NULL == g2g_runtime || NULL == g2g_schema || NULL == line) {
if (NULL == g2g_runtime || NULL == g2g_schema ||
NULL == line) {
return -1;
}
@@ -501,10 +542,12 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *
if (0 == is_valid) {
//delete
ret = group2group_runtime_remove_group_from_group(g2g_runtime, g2g_item->group_id, g2g_item->superior_group_id);
ret = group2group_runtime_remove_group_from_group(g2g_runtime, g2g_item->group_id,
g2g_item->super_group_id);
} else {
//add
ret = group2group_runtime_add_group_to_group(g2g_runtime, g2g_item->group_id, g2g_item->superior_group_id);
ret = group2group_runtime_add_group_to_group(g2g_runtime, g2g_item->group_id,
g2g_item->super_group_id);
}
group2group_item_free(g2g_item);
@@ -520,8 +563,8 @@ int group2group_runtime_commit(void *g2g_runtime)
return group2group_runtime_build_top_groups(g2g_runtime);
}
int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t n_group_ids,
int *top_group_ids)
int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids,
size_t n_group_ids, int *top_group_ids)
{
if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) {
return -1;
@@ -540,4 +583,4 @@ int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t
}
return top_group_index;
}
}

529
src/maat_ip.cpp
View File

@@ -11,16 +11,15 @@
#include <stdint.h>
#include <assert.h>
#include "utils.h"
#include "log/log.h"
#include "cJSON/cJSON.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_ex_data.h"
#include "IPMatcher.h"
#include "maat_ip.h"
#include "maat_rule.h"
#include "maat_compile.h"
#include "alignment.h"
#include "maat_garbage_collection.h"
#define MODULE_IP module_name_str("maat.ip")
@@ -30,7 +29,7 @@ struct port_range {
uint16_t max_port;
};
struct ip_plus_schema {
struct ip_schema {
int item_id_column;
int group_id_column;
int addr_type_column;
@@ -40,15 +39,10 @@ struct ip_plus_schema {
int sport_format_column;
int sport1_column;
int sport2_column;
int daddr_format_column;
int dip1_column;
int dip2_column;
int dport_format_column;
int dport1_column;
int dport2_column;
int proto_column;
int direction_column;
int table_id; //ugly
struct table_manager *tbl_mgr;
};
struct ipv4_item_rule {
@@ -69,7 +63,7 @@ struct ipv6_item_rule {
uint16_t direction; /* 方向，0表示双向，1表示单向 */
};
struct ip_plus_item {
struct ip_item {
int item_id;
int group_id;
int addr_type;
@@ -79,7 +73,7 @@ struct ip_plus_item {
};
};
struct ip_plus_runtime {
struct ip_runtime {
struct ip_matcher* ip_matcher;
struct ex_data_runtime* ex_data_rt;
@@ -90,187 +84,174 @@ struct ip_plus_runtime {
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long *scan_cnt;
long long *hit_cnt;
};
void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
size_t read_cnt = 0;
struct ip_plus_schema *ip_plus_schema = ALLOC(struct ip_plus_schema, 1);
struct ip_schema *ip_schema = ALLOC(struct ip_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
ip_plus_schema->table_id = item->valueint;
ip_schema->table_id = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_IP, "ip_plus table %s has no custom column", table_name);
log_error(logger, MODULE_IP,
"ip table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->item_id_column = custom_item->valueint;
ip_schema->item_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->group_id_column = custom_item->valueint;
ip_schema->group_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "addr_type");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->addr_type_column = custom_item->valueint;
ip_schema->addr_type_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "saddr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->saddr_format_column = custom_item->valueint;
ip_schema->saddr_format_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "sip1");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sip1_column = custom_item->valueint;
ip_schema->sip1_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "sip2");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sip2_column = custom_item->valueint;
ip_schema->sip2_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "sport_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport_format_column = custom_item->valueint;
ip_schema->sport_format_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "sport1");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport1_column = custom_item->valueint;
ip_schema->sport1_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "sport2");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport2_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "daddr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->daddr_format_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "dip1");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dip1_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "dip2");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dip2_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "dport_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport_format_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "dport1");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport1_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "dport2");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport2_column = custom_item->valueint;
ip_schema->sport2_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "proto");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->proto_column = custom_item->valueint;
ip_schema->proto_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "direction");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->direction_column = custom_item->valueint;
ip_schema->direction_column = custom_item->valueint;
read_cnt++;
}
if (read_cnt < 18) {
ip_schema->tbl_mgr = tbl_mgr;
if (read_cnt < 12) {
goto error;
}
return ip_plus_schema;
return ip_schema;
error:
FREE(ip_plus_schema);
FREE(ip_schema);
return NULL;
}
void ip_plus_schema_free(void *ip_plus_schema)
void ip_schema_free(void *ip_schema)
{
FREE(ip_plus_schema);
FREE(ip_schema);
}
void *ip_plus_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
void *ip_runtime_new(void *ip_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == ip_plus_schema) {
if (NULL == ip_schema) {
return NULL;
}
struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema;
struct ip_plus_runtime *ip_plus_rt = ALLOC(struct ip_plus_runtime, 1);
struct ip_schema *schema = (struct ip_schema *)ip_schema;
struct ip_runtime *ip_rt = ALLOC(struct ip_runtime, 1);
ip_plus_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger);
ip_plus_rt->item_user_data_free = maat_item_inner_free;
ip_plus_rt->ref_garbage_bin = garbage_bin;
ip_plus_rt->logger = logger;
ip_rt->ex_data_rt = ex_data_runtime_new(schema->table_id,
ex_data_container_free,
logger);
ip_rt->item_user_data_free = maat_item_inner_free;
ip_rt->ref_garbage_bin = garbage_bin;
ip_rt->logger = logger;
return ip_plus_rt;
ip_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
ip_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
return ip_rt;
}
void ip_plus_runtime_free(void *ip_plus_runtime)
void ip_runtime_free(void *ip_runtime)
{
if (NULL == ip_plus_runtime) {
if (NULL == ip_runtime) {
return;
}
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime;
if (ip_plus_rt->ip_matcher != NULL) {
ip_matcher_free(ip_plus_rt->ip_matcher);
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
if (ip_rt->ip_matcher != NULL) {
ip_matcher_free(ip_rt->ip_matcher);
}
if (ip_plus_rt->ex_data_rt != NULL) {
ex_data_runtime_free(ip_plus_rt->ex_data_rt);
if (ip_rt->ex_data_rt != NULL) {
ex_data_runtime_free(ip_rt->ex_data_rt);
}
struct maat_item *item = NULL, *tmp_item = NULL;
HASH_ITER(hh, ip_plus_rt->item_hash, item, tmp_item) {
HASH_DELETE(hh, ip_plus_rt->item_hash, item);
maat_item_free(item, ip_plus_rt->item_user_data_free);
HASH_ITER(hh, ip_rt->item_hash, item, tmp_item) {
HASH_DELETE(hh, ip_rt->item_hash, item);
maat_item_free(item, ip_rt->item_user_data_free);
}
FREE(ip_plus_rt);
if (ip_rt->hit_cnt != NULL) {
alignment_int64_array_free(ip_rt->hit_cnt);
ip_rt->hit_cnt = NULL;
}
if (ip_rt->scan_cnt != NULL) {
alignment_int64_array_free(ip_rt->scan_cnt);
ip_rt->scan_cnt = NULL;
}
FREE(ip_rt);
}
struct ip_plus_item *ip_plus_item_new(const char *line, struct ip_plus_schema *ip_plus_schema,
struct log_handle *logger)
struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
@@ -282,198 +263,209 @@ struct ip_plus_item *ip_plus_item_new(const char *line, struct ip_plus_schema *i
uint16_t sport2 = 0;
uint16_t protocol = 0;
uint16_t direction = 0;
struct ip_plus_item *ip_plus_item = ALLOC(struct ip_plus_item, 1);
struct ip_item *ip_item = ALLOC(struct ip_item, 1);
int ret = get_column_pos(line, ip_plus_schema->item_id_column, &column_offset, &column_len);
int ret = get_column_pos(line, ip_schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no item_id",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP,
"ip plus table(table_id:%d) line:%s has no item_id",
ip_schema->table_id, line);
goto error;
}
ip_plus_item->item_id = atoi(line + column_offset);
ip_item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, ip_plus_schema->group_id_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no group_id",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP,
"ip plus table(table_id:%d) line:%s has no group_id",
ip_schema->table_id, line);
goto error;
}
ip_plus_item->group_id = atoi(line + column_offset);
ip_item->group_id = atoi(line + column_offset);
ret = get_column_pos(line, ip_plus_schema->addr_type_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->addr_type_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no addr_type",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP,
"ip plus table(table_id:%d) line:%s has no addr_type",
ip_schema->table_id, line);
goto error;
}
ip_plus_item->addr_type = atoi(line + column_offset);
ip_item->addr_type = atoi(line + column_offset);
if (ip_plus_item->addr_type != 4 && ip_plus_item->addr_type != 6) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has invalid addr type:%d",
ip_plus_schema->table_id, line, ip_plus_item->addr_type);
if (ip_item->addr_type != IPv4 && ip_item->addr_type != IPv6) {
log_error(logger, MODULE_IP,
"ip table(table_id:%d) line:%s has invalid addr type:%d",
ip_schema->table_id, line, ip_item->addr_type);
goto error;
}
ret = get_column_pos(line, ip_plus_schema->saddr_format_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->saddr_format_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no saddr_format",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP,
"ip table(table_id:%d) line:%s has no saddr_format",
ip_schema->table_id, line);
goto error;
}
memcpy(saddr_format, (line + column_offset), column_len);
if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) {
log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid saddr_format, should be range/mask/CIDR",
ip_plus_schema->table_id, line);
"ip table(table_id:%d) line:%s has invalid saddr_format, should be range/mask/CIDR",
ip_schema->table_id, line);
goto error;
}
ret = get_column_pos(line, ip_plus_schema->sip1_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->sip1_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sip1",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no sip1",
ip_schema->table_id, line);
goto error;
}
memcpy(sip1_str, (line + column_offset), column_len);
ret = get_column_pos(line, ip_plus_schema->sip2_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->sip2_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sip2",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no sip2",
ip_schema->table_id, line);
goto error;
}
memcpy(sip2_str, (line + column_offset), column_len);
ret = get_column_pos(line, ip_plus_schema->sport_format_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->sport_format_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport_format",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP,
"ip table(table_id:%d) line:%s has no sport_format",
ip_schema->table_id, line);
goto error;
}
memcpy(sport_format, (line + column_offset), column_len);
if (IP_FORMAT_UNKNOWN == ip_format_str2int(sport_format)) {
log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid sport_format, should be range/mask/CIDR",
ip_plus_schema->table_id, line);
"ip table(table_id:%d) line:%s has invalid sport_format, should be range/mask/CIDR",
ip_schema->table_id, line);
goto error;
}
ret = get_column_pos(line, ip_plus_schema->sport1_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->sport1_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport1",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no sport1",
ip_schema->table_id, line);
goto error;
}
sport1 = atoi(line + column_offset);
ret = get_column_pos(line, ip_plus_schema->sport2_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->sport2_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport2",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP,
"ip table(table_id:%d) line:%s has no sport2",
ip_schema->table_id, line);
goto error;
}
sport2 = atoi(line + column_offset);
if (4 == ip_plus_item->addr_type) {
ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str,
&ip_plus_item->ipv4.min_sip, &ip_plus_item->ipv4.max_sip);
if (IPv4 == ip_item->addr_type) {
ret = ip_format2range(ip_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str,
&ip_item->ipv4.min_sip, &ip_item->ipv4.max_sip);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s ip_format2range(ip4) failed",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP,
"ip table(table_id:%d) line:%s ip_format2range(ip4) failed",
ip_schema->table_id, line);
goto error;
}
if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) {
ip_plus_item->ipv4.min_sport = sport1 & sport2;
ip_plus_item->ipv4.max_sport = sport1 | ~sport2;
ip_item->ipv4.min_sport = sport1 & sport2;
ip_item->ipv4.max_sport = sport1 | ~sport2;
} else {
ip_plus_item->ipv4.min_sport = sport1;
ip_plus_item->ipv4.max_sport = sport2;
ip_item->ipv4.min_sport = sport1;
ip_item->ipv4.max_sport = sport2;
}
ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->proto_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no proto",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP,
"ip table(table_id:%d) line:%s has no proto",
ip_schema->table_id, line);
goto error;
}
ip_plus_item->ipv4.proto = atoi(line + column_offset);
protocol = ip_plus_item->ipv4.proto;
ip_item->ipv4.proto = atoi(line + column_offset);
protocol = ip_item->ipv4.proto;
ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->direction_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no direction",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP,
"ip table(table_id:%d) line:%s has no direction",
ip_schema->table_id, line);
goto error;
}
ip_plus_item->ipv4.direction = atoi(line + column_offset);
direction = ip_plus_item->ipv4.direction;
ip_item->ipv4.direction = atoi(line + column_offset);
direction = ip_item->ipv4.direction;
} else {
//ipv6
ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str,
ip_plus_item->ipv6.min_sip, ip_plus_item->ipv6.max_sip);
ret = ip_format2range(ip_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str,
ip_item->ipv6.min_sip, ip_item->ipv6.max_sip);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s ip_format2range(ip6) failed",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP,
"ip table(table_id:%d) line:%s ip_format2range(ip6) failed",
ip_schema->table_id, line);
goto error;
}
if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) {
ip_plus_item->ipv6.min_sport = sport1 & sport2;
ip_plus_item->ipv6.max_sport = sport1 | ~sport2;
ip_item->ipv6.min_sport = sport1 & sport2;
ip_item->ipv6.max_sport = sport1 | ~sport2;
} else {
ip_plus_item->ipv6.min_sport = sport1;
ip_plus_item->ipv6.max_sport = sport2;
ip_item->ipv6.min_sport = sport1;
ip_item->ipv6.max_sport = sport2;
}
ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->proto_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no proto",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no proto",
ip_schema->table_id, line);
goto error;
}
ip_plus_item->ipv6.proto = atoi(line + column_offset);
protocol = ip_plus_item->ipv6.proto;
ip_item->ipv6.proto = atoi(line + column_offset);
protocol = ip_item->ipv6.proto;
ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->direction_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no direction",
ip_plus_schema->table_id, line);
log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no direction",
ip_schema->table_id, line);
goto error;
}
ip_plus_item->ipv6.direction = atoi(line + column_offset);
direction = ip_plus_item->ipv6.direction;
ip_item->ipv6.direction = atoi(line + column_offset);
direction = ip_item->ipv6.direction;
}
if (protocol > 65535 || protocol < 0) {
log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid proto:%d",
ip_plus_schema->table_id, line, protocol);
"ip table(table_id:%d) line:%s has invalid proto:%d",
ip_schema->table_id, line, protocol);
goto error;
}
if (direction != 0 && direction != 1) {
log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid direction:%d",
ip_plus_schema->table_id, line, direction);
"ip table(table_id:%d) line:%s has invalid direction:%d",
ip_schema->table_id, line, direction);
goto error;
}
return ip_plus_item;
return ip_item;
error:
FREE(ip_plus_item);
FREE(ip_item);
return NULL;
}
void ip_plus_item_free(struct ip_plus_item *ip_plus_item)
void ip_item_free(struct ip_item *ip_item)
{
FREE(ip_plus_item);
FREE(ip_item);
}
void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule)
void ip_item_to_ip_rule(struct ip_item *item, struct ip_rule *rule)
{
struct port_range *sport_range = ALLOC(struct port_range, 1);
if (4 == item->addr_type) {
if (IPv4 == item->addr_type) {
rule->type = IPv4;
sport_range->min_port = item->ipv4.min_sport;
sport_range->max_port = item->ipv4.max_sport;
@@ -483,20 +475,22 @@ void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule)
rule->type = IPv6;
sport_range->min_port = item->ipv6.min_sport;
sport_range->max_port = item->ipv6.max_sport;
memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, sizeof(item->ipv6.min_sip));
memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, sizeof(item->ipv6.max_sip));
memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip,
sizeof(item->ipv6.min_sip));
memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip,
sizeof(item->ipv6.max_sip));
}
rule->rule_id = item->item_id;
rule->user_tag = sport_range;
}
struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt)
struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt)
{
return ip_plus_rt->ex_data_rt;
return ip_rt->ex_data_rt;
}
int ip_plus_runtime_update_row(struct ip_plus_runtime *rt, char *key, size_t key_len,
struct ip_plus_item *item, int is_valid)
int ip_runtime_update_row(struct ip_runtime *rt, char *key, size_t key_len,
struct ip_item *item, int is_valid)
{
int ret = -1;
struct ex_data_runtime *ex_data_rt = rt->ex_data_rt;
@@ -519,31 +513,25 @@ int ip_plus_runtime_update_row(struct ip_plus_runtime *rt, char *key, size_t key
return 0;
}
int ip_plus_runtime_updating_flag(void *ip_plus_runtime)
int ip_runtime_update(void *ip_runtime, void *ip_schema,
const char *line, int valid_column)
{
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime;
return ex_data_runtime_updating_flag(ip_plus_rt->ex_data_rt);
}
int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line,
int valid_column)
{
if (NULL == ip_plus_runtime || NULL == ip_plus_schema || NULL == line) {
if (NULL == ip_runtime || NULL == ip_schema || NULL == line) {
return -1;
}
struct maat_item *item = NULL;
struct ip_plus_item *ip_plus_item = NULL;
struct ip_item *ip_item = NULL;
struct maat_item_inner *u_para = NULL;
struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema;
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime;
struct ip_schema *schema = (struct ip_schema *)ip_schema;
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND_INT(ip_plus_rt->item_hash, &item_id, item);
HASH_FIND_INT(ip_rt->item_hash, &item_id, item);
if (NULL == item) {
return -1;
}
@@ -555,56 +543,58 @@ int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const ch
return -1;
}
HASH_DELETE(hh, ip_plus_rt->item_hash, item);
maat_garbage_bagging(ip_plus_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free);
HASH_DELETE(hh, ip_rt->item_hash, item);
maat_garbage_bagging(ip_rt->ref_garbage_bin, u_para,
(void (*)(void *))maat_item_inner_free);
} else {
//add
HASH_FIND_INT(ip_plus_rt->item_hash, &item_id, item);
HASH_FIND_INT(ip_rt->item_hash, &item_id, item);
if (item) {
log_error(ip_plus_rt->logger, MODULE_IP,
"ip_plus runtime add item %d to item_hash failed, already exist", item_id);
log_error(ip_rt->logger, MODULE_IP,
"ip runtime add item %d to item_hash failed, already exist",
item_id);
return -1;
}
ip_plus_item = ip_plus_item_new(line, schema, ip_plus_rt->logger);
if (NULL == ip_plus_item) {
ip_item = ip_item_new(line, schema, ip_rt->logger);
if (NULL == ip_item) {
return -1;
}
u_para = maat_item_inner_new(ip_plus_item->group_id, item_id, 0);
item = maat_item_new(item_id, ip_plus_item->group_id, u_para);
HASH_ADD_INT(ip_plus_rt->item_hash, item_id, item);
u_para = maat_item_inner_new(ip_item->group_id, item_id, 0);
item = maat_item_new(item_id, ip_item->group_id, u_para);
HASH_ADD_INT(ip_rt->item_hash, item_id, item);
}
char *key = (char *)&item_id;
int ret = ip_plus_runtime_update_row(ip_plus_rt, key, sizeof(int), ip_plus_item, is_valid);
int ret = ip_runtime_update_row(ip_rt, key, sizeof(int), ip_item, is_valid);
if (ret < 0) {
if (ip_plus_item != NULL) {
ip_plus_item_free(ip_plus_item);
ip_plus_item = NULL;
if (ip_item != NULL) {
ip_item_free(ip_item);
ip_item = NULL;
}
return -1;
} else {
if (0 == is_valid) {
ip_plus_rt->rule_num--;
ip_rt->rule_num--;
} else {
ip_plus_rt->rule_num++;
ip_rt->rule_num++;
}
}
return 0;
}
int ip_plus_runtime_commit(void *ip_plus_runtime)
int ip_runtime_commit(void *ip_runtime)
{
if (NULL == ip_plus_runtime) {
if (NULL == ip_runtime) {
return -1;
}
int ret = 0;
struct ex_data_container **ex_container = NULL;
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime;
struct ex_data_runtime *ex_data_rt = ip_plus_rt->ex_data_rt;
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
struct ex_data_runtime *ex_data_rt = ip_rt->ex_data_rt;
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (0 == rule_cnt) {
@@ -615,8 +605,8 @@ int ip_plus_runtime_commit(void *ip_plus_runtime)
struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
struct ip_plus_item *item = (struct ip_plus_item *)ex_container[i]->custom_data;
ip_plus_item_to_ip_rule(item, &rules[i]);
struct ip_item *item = (struct ip_item *)ex_container[i]->custom_data;
ip_item_to_ip_rule(item, &rules[i]);
}
struct ip_matcher *new_ip_matcher = NULL;
@@ -624,21 +614,23 @@ int ip_plus_runtime_commit(void *ip_plus_runtime)
size_t mem_used = 0;
if (rule_cnt > 0) {
log_info(ip_plus_rt->logger, MODULE_IP,
"committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt);
log_info(ip_rt->logger, MODULE_IP,
"committing %zu ip rules for rebuilding ip_matcher engine",
rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
log_error(ip_plus_rt->logger, MODULE_IP,
"rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt);
log_error(ip_rt->logger, MODULE_IP,
"rebuild ip_matcher engine failed when update %zu ip rules",
rule_cnt);
ret = -1;
}
}
old_ip_matcher = ip_plus_rt->ip_matcher;
ip_plus_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(ip_plus_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
old_ip_matcher = ip_rt->ip_matcher;
ip_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(ip_rt->ref_garbage_bin, old_ip_matcher,
(void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ex_data_rt);
ip_plus_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt);
FREE(rules);
FREE(ex_container);
@@ -646,66 +638,57 @@ int ip_plus_runtime_commit(void *ip_plus_runtime)
return ret;
}
int ip_plus_runtime_scan_ipv4(struct ip_plus_runtime *ip_plus_rt, int thread_id, uint32_t ip_addr,
int *group_id_array, size_t n_group_id_array, int virtual_table_id,
struct maat_state *state)
int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, int ip_type,
uint8_t *ip_addr, int *group_ids, size_t group_id_size,
int vtable_id, struct maat_state *state)
{
if (NULL == ip_plus_rt) {
return -1;
}
int n_hit_item = 0;
struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct ip_data ip;
ip.type = IPv4;
ip.ipv4 = ip_addr;
n_hit_item = ip_matcher_match(ip_plus_rt->ip_matcher, &ip, scan_results, MAX_SCANNER_HIT_ITEM_NUM);
struct ip_data scan_data;
if (ip_type == IPv4) {
scan_data.type = IPv4;
scan_data.ipv4 = *(uint32_t *)ip_addr;
} else {
scan_data.type = IPv6;
for (int i = 0; i < 4; i++) {
scan_data.ipv6[i] = *((uint32_t *)ip_addr + i);
}
}
n_hit_item = ip_matcher_match(ip_rt->ip_matcher, &scan_data, scan_results, sizeof(scan_result));
if (n_hit_item <= 0) {
return n_hit_item;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(ip_rt->logger, MODULE_IP,
"hit ip item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_compile_state *compile_state = state->compile_mid;
//tranform item_id to group_id
struct maat_item *item = NULL;
size_t n_group_id = 0;
int i = 0;
for (i = 0; i < n_hit_item; i++) {
HASH_FIND_INT(ip_plus_rt->item_hash, &(scan_results[i].rule_id), item);
assert(item != NULL);
if (!item) {
// should not come here
continue;
}
if (n_group_id >= n_group_id_array) {
n_group_id = n_group_id_array;
//Prevent group_id_array out of bounds
} else {
group_id_array[n_group_id++] = item->group_id;
}
// update hit path
maat_compile_state_update_hit_path(compile_state, scan_results[i].rule_id, item->group_id,
virtual_table_id, state->scan_cnt, i);
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = scan_results[i].rule_id;
}
// update hit clause: literal_id{group_id,vt_id} to clause_id
int compile_table_id = -1;
if (state->compile_table_id == -1) {
compile_table_id = state->maat_instance->default_compile_table_id;
} else {
compile_table_id = state->compile_table_id;
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item,
group_ids, group_id_size, &group_hit_cnt, state);
if (ret < 0) {
return -1;
}
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id);
enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, compile_table_id);
assert(table_type == TABLE_TYPE_COMPILE);
for (size_t idx = 0; idx < n_group_id; idx++) {
maat_compile_state_update_hit_clause(compile_state, compile_rt, group_id_array[idx], virtual_table_id);
}
return group_hit_cnt;
}
return n_group_id;
void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id)
{
alignment_int64_array_add(ip_rt->hit_cnt, thread_id, 1);
}
long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread)
{
return alignment_int64_array_sum(ip_rt->hit_cnt, n_thread);
}

129
src/maat_ip_plugin.cpp
View File

@@ -10,9 +10,8 @@
#include <assert.h>
#include "maat_ip_plugin.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_ip_plugin.h"
#include "maat_ex_data.h"
#include "IPMatcher.h"
#include "maat_rule.h"
@@ -37,6 +36,10 @@ struct ip_plugin_schema {
int rule_tag_column;
struct ex_data_schema *ex_schema;
int table_id; //ugly
struct table_manager *tbl_mgr;
unsigned long long update_err_cnt;
unsigned long long unmatch_tag_cnt;
};
struct ip_plugin_runtime {
@@ -50,7 +53,8 @@ struct ip_plugin_runtime {
struct log_handle *logger;
};
void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
size_t read_cnt = 0;
struct ip_plugin_schema *ip_plugin_schema = ALLOC(struct ip_plugin_schema, 1);
@@ -64,7 +68,8 @@ void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handl
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_IP_PLUGIN, "table %s has no custom column", table_name);
log_error(logger, MODULE_IP_PLUGIN,
"table %s has no custom column", table_name);
goto error;
}
@@ -92,6 +97,8 @@ void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handl
read_cnt++;
}
ip_plugin_schema->tbl_mgr = tbl_mgr;
if (read_cnt < 5) {
goto error;
}
@@ -127,52 +134,102 @@ struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema
return schema->ex_schema;
}
struct ip_plugin_item *ip_plugin_item_new(const char *line, struct ip_plugin_schema *ip_plugin_schema,
struct log_handle *logger)
int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
size_t n_tag = table_manager_accept_tags_count(schema->tbl_mgr);
if (schema->rule_tag_column > 0 && n_tag > 0) {
int ret = get_column_pos(line, schema->rule_tag_column,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) has no rule_tag, line:%s",
schema->table_id, line);
schema->update_err_cnt++;
return TAG_MATCH_ERR;
}
if (column_len > 2) {
char *tag_str = ALLOC(char, column_len + 1);
memcpy(tag_str, (line + column_offset), column_len);
ret = table_manager_accept_tags_match(schema->tbl_mgr, tag_str);
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) has invalid tag format, line:%s",
schema->table_id, line);
schema->update_err_cnt++;
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
schema->unmatch_tag_cnt++;
return TAG_MATCH_UNMATCHED;
}
}
}
return TAG_MATCH_MATCHED;
}
struct ip_plugin_item *
ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
struct log_handle *logger)
{
int ret = ip_plugin_accept_tag_match(schema, line, logger);
if (ret == TAG_MATCH_UNMATCHED) {
return NULL;
}
size_t column_offset = 0;
size_t column_len = 0;
struct ip_plugin_item *ip_plugin_item = ALLOC(struct ip_plugin_item, 1);
int ret = get_column_pos(line, ip_plugin_schema->item_id_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, "ip plugin table(table_id:%d) line:%s has no item_id",
ip_plugin_schema->table_id, line);
log_error(logger, MODULE_IP_PLUGIN,
"ip plugin table(table_id:%d) line:%s has no item_id",
schema->table_id, line);
goto error;
}
ip_plugin_item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, ip_plugin_schema->ip_type_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->ip_type_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, "ip plugin table(table_id:%d) line:%s has no ip_type",
ip_plugin_schema->table_id, line);
log_error(logger, MODULE_IP_PLUGIN,
"ip plugin table(table_id:%d) line:%s has no ip_type",
schema->table_id, line);
goto error;
}
ip_plugin_item->ip_type = atoi(line + column_offset);
if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s ip_type[%d] invalid",
ip_plugin_schema->table_id, line, ip_plugin_item->ip_type);
schema->table_id, line, ip_plugin_item->ip_type);
goto error;
}
ret = get_column_pos(line, ip_plugin_schema->start_ip_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s has no start_ip",
ip_plugin_schema->table_id, line);
schema->table_id, line);
goto error;
}
strncpy(ip_plugin_item->start_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->start_ip)));
strncpy(ip_plugin_item->start_ip, line + column_offset,
MIN(column_len, sizeof(ip_plugin_item->start_ip)));
ret = get_column_pos(line, ip_plugin_schema->end_ip_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->end_ip_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s has no end_ip",
ip_plugin_schema->table_id, line);
schema->table_id, line);
goto error;
}
strncpy(ip_plugin_item->end_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->end_ip)));
strncpy(ip_plugin_item->end_ip, line + column_offset,
MIN(column_len, sizeof(ip_plugin_item->end_ip)));
return ip_plugin_item;
error:
@@ -245,7 +302,8 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_
return 0;
}
void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == ip_plugin_schema) {
@@ -255,7 +313,8 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct m
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1);
ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger);
ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id,
ex_data_container_free, logger);
ip_plugin_rt->ref_garbage_bin = garbage_bin;
ip_plugin_rt->logger = logger;
@@ -298,10 +357,11 @@ void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule
rule->user_tag = NULL;
}
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line,
int valid_column)
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
const char *line, int valid_column)
{
if (NULL == ip_plugin_runtime || NULL == ip_plugin_schema || NULL == line) {
if (NULL == ip_plugin_runtime || NULL == ip_plugin_schema ||
NULL == line) {
return -1;
}
@@ -323,7 +383,8 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, co
}
char *key = (char *)&item_id;
int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, line, key, sizeof(int), ip_plugin_item, is_valid);
int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, line, key,
sizeof(int), ip_plugin_item, is_valid);
if (ret < 0) {
if (ip_plugin_item != NULL) {
FREE(ip_plugin_item);
@@ -370,20 +431,22 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime)
if (rule_cnt > 0) {
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt);
"committing %zu ip_plugin rules for rebuilding ip_matcher engine",
rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt);
"rebuild ip_matcher engine failed when update %zu ip_plugin rules",
rule_cnt);
ret = -1;
}
}
old_ip_matcher = ip_plugin_rt->ip_matcher;
ip_plugin_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher,
(void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ex_data_rt);
ip_plugin_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt);
FREE(rules);
FREE(ex_container);
@@ -391,12 +454,6 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime)
return ret;
}
int ip_plugin_runtime_updating_flag(void *ip_plugin_runtime)
{
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt);
}
struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime)
{
if (NULL == ip_plugin_runtime) {

2
src/maat_kv.cpp
View File

@@ -12,7 +12,7 @@
#include <ctype.h>
#include "uthash/uthash.h"
#include "utils.h"
#include "maat_utils.h"
#define MAAT_KV_MAX_KEY_LEN 512

117
src/maat_plugin.cpp
View File

@@ -12,11 +12,12 @@
#include "log/log.h"
#include "cJSON/cJSON.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_rule.h"
#include "maat_plugin.h"
#include "maat_ex_data.h"
#include "maat_limits.h"
#include "maat_table.h"
#define MODULE_PLUGIN module_name_str("maat.plugin")
@@ -49,6 +50,10 @@ struct plugin_schema {
struct plugin_callback_schema cb[MAX_PLUGIN_PER_TABLE];
struct ex_data_schema *ex_schema;
int table_id; //ugly
struct table_manager *tbl_mgr;
unsigned long long update_err_cnt;
unsigned long long unmatch_tag_cnt;
};
static int read_integer_array(char *string, int *array, int size)
@@ -68,7 +73,8 @@ static int read_integer_array(char *string, int *array, int size)
return i;
}
void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
size_t read_cnt = 0;
struct plugin_schema *plugin_schema = ALLOC(struct plugin_schema, 1);
@@ -82,7 +88,8 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_PLUGIN, "table %s has no custom column", table_name);
log_error(logger, MODULE_PLUGIN,
"table %s has no custom column", table_name);
goto error;
}
@@ -121,6 +128,8 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *
}
}
plugin_schema->tbl_mgr = tbl_mgr;
if (read_cnt < 5) {
goto error;
}
@@ -160,8 +169,9 @@ int plugin_table_add_callback(void *plugin_schema, int table_id,
size_t idx = schema->cb_cnt;
if (idx == MAX_PLUGIN_PER_TABLE) {
log_error(logger, MODULE_PLUGIN, "the plugin number of table_id: %d exceed maxium:%d",
table_id, MAX_PLUGIN_PER_TABLE);
log_error(logger, MODULE_PLUGIN,
"the plugin number of table_id: %d exceed maxium:%d",
table_id, MAX_PLUGIN_PER_TABLE);
return -1;
}
@@ -192,7 +202,8 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema)
}
}
int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns)
int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema,
int *foreign_columns)
{
if (NULL == plugin_schema) {
return -1;
@@ -221,7 +232,8 @@ int plugin_table_set_ex_data_schema(void *plugin_schema,
if (schema->ex_schema != NULL) {
assert(0);
log_error(logger, MODULE_PLUGIN,
"Error: %s, EX data schema already registed", __FUNCTION__);
"Error: %s, EX data schema already registed",
__FUNCTION__);
return -1;
}
schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp);
@@ -239,7 +251,8 @@ struct ex_data_schema *plugin_table_get_ex_data_schema(void *plugin_schema)
return schema->ex_schema;
}
void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
void *plugin_runtime_new(void *plugin_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == plugin_schema) {
@@ -248,7 +261,8 @@ void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_ga
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
struct plugin_runtime *plugin_rt = ALLOC(struct plugin_runtime, 1);
plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger);
plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id,
ex_data_container_free, logger);
plugin_rt->ref_garbage_bin = garbage_bin;
plugin_rt->logger = logger;
@@ -270,18 +284,12 @@ void plugin_runtime_free(void *plugin_runtime)
FREE(plugin_rt);
}
int plugin_runtime_updating_flag(void *plugin_runtime)
{
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt);
}
int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_schema *plugin_schema,
const char *row, char *key, size_t key_len, int is_valid)
int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
struct plugin_schema *plugin_schema,
const char *row, char *key, size_t key_len,
int is_valid)
{
int ret = -1;
void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL);
struct ex_data_schema *ex_schema = plugin_schema->ex_schema;
/* already set plugin_table_schema's ex_data_schema */
@@ -294,6 +302,8 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_sc
}
} else {
// add
void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL);
ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
return -1;
@@ -313,33 +323,77 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_sc
ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row);
}
plugin_rt->acc_line_num++;
return 0;
}
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line,
int valid_column)
int plugin_accept_tag_match(struct plugin_schema *schema, const char *line,
struct log_handle *logger)
{
if (NULL == plugin_runtime || NULL == plugin_runtime) {
size_t column_offset = 0;
size_t column_len = 0;
size_t n_tag = table_manager_accept_tags_count(schema->tbl_mgr);
if (schema->rule_tag_column > 0 && n_tag > 0) {
int ret = get_column_pos(line, schema->rule_tag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_PLUGIN,
"plugin table(table_id:%d) has no rule_tag, line:%s",
schema->table_id, line);
schema->update_err_cnt++;
return TAG_MATCH_ERR;
}
if (column_len > 2) {
char *tag_str = ALLOC(char, column_len + 1);
memcpy(tag_str, (line + column_offset), column_len);
ret = table_manager_accept_tags_match(schema->tbl_mgr, tag_str);
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_PLUGIN,
"plugin table(table_id:%d) has invalid tag format, line:%s",
schema->table_id, line);
schema->update_err_cnt++;
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
schema->unmatch_tag_cnt++;
return TAG_MATCH_UNMATCHED;
}
}
}
return TAG_MATCH_MATCHED;
}
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
const char *line, int valid_column)
{
if (NULL == plugin_runtime || NULL == plugin_schema) {
return -1;
}
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
}
char *key = (char *)&item_id;
int ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid);
if (ret < 0) {
} else {
int ret = plugin_accept_tag_match(schema, line, plugin_rt->logger);
if (ret == TAG_MATCH_UNMATCHED) {
return -1;
}
int item_id = get_column_value(line, schema->item_id_column);
char *key = (char *)&item_id;
ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid);
if (ret < 0) {
schema->update_err_cnt++;
return -1;
}
plugin_rt->acc_line_num++;
return 0;
}
@@ -351,7 +405,6 @@ int plugin_runtime_commit(void *plugin_runtime)
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
ex_data_runtime_commit(plugin_rt->ex_data_rt);
//table_rt->rule_num = ex_data_runtime_ex_container_count(table_rt->plugin_rt.ex_data_rt);
return 0;
}

323
src/maat_redis_monitor.cpp
View File

@@ -11,8 +11,9 @@
#include <string.h>
#include <assert.h>
#include <sys/stat.h>
#include <stdio.h>
#include "utils.h"
#include "maat/maat.h"
#include "maat_utils.h"
#include "maat_command.h"
#include "maat_config_monitor.h"
@@ -39,28 +40,34 @@ const char *foreign_key_prefix = "__FILE_";
const char *mr_op_str[] = {"DEL", "ADD", "RENEW_TIMEOUT"};
char *get_foreign_cont_filename(const char *table_name, int rule_id, const char *foreign_key, const char *dir)
char *get_foreign_cont_filename(const char *table_name, int rule_id,
const char *foreign_key, const char *dir)
{
char buffer[512] = {0};
snprintf(buffer, sizeof(buffer),"%s/%s-%d-%s", dir, table_name, rule_id, foreign_key);
snprintf(buffer, sizeof(buffer),"%s/%s-%d-%s", dir,
table_name, rule_id, foreign_key);
char *filename = ALLOC(char, strlen(buffer) + 1);
memcpy(filename, buffer, strlen(buffer));
return filename;
}
void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_foreign,
const char *dir, struct log_handle *logger)
void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns,
int n_foreign, const char *dir, struct log_handle *logger)
{
int foreign_key_size = 0;
p_rule->f_keys = ALLOC(struct foreign_key, n_foreign);
for (int i = 0; i < n_foreign; i++) {
const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line, foreign_columns[i], &foreign_key_size);
const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line,
foreign_columns[i],
&foreign_key_size);
if (NULL == p_foreign) {
log_error(logger, MODULE_REDIS_MONITOR, "Get %s,%lu foreign keys failed: No %dth column",
p_rule->table_name, p_rule->rule_id, foreign_columns[i]);
log_error(logger, MODULE_REDIS_MONITOR,
"Get %s,%lu foreign keys failed: No %dth column",
p_rule->table_name, p_rule->rule_id,
foreign_columns[i]);
continue;
}
@@ -70,7 +77,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
}
if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) {
log_error(logger, MODULE_REDIS_MONITOR, "Get %s,%lu foreign key failed: Invalid source prefix %s",
log_error(logger, MODULE_REDIS_MONITOR,
"Get %s,%lu foreign key failed: Invalid source prefix %s",
p_rule->table_name, p_rule->rule_id, p_foreign);
continue;
}
@@ -80,13 +88,17 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
p_foreign += strlen(foreign_source_prefix);
if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) {
log_info(logger, MODULE_REDIS_MONITOR, "%s, %lu foreign key prefix %s is not recommended",
log_info(logger, MODULE_REDIS_MONITOR,
"%s, %lu foreign key prefix %s is not recommended",
p_rule->table_name, p_rule->rule_id, p_foreign);
}
p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size+1);
memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign, foreign_key_size);
p_rule->f_keys[p_rule->n_foreign].filename = get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[p_rule->n_foreign].key, dir);
p_rule->f_keys[p_rule->n_foreign].filename = get_foreign_cont_filename(p_rule->table_name,
p_rule->rule_id,
p_rule->f_keys[p_rule->n_foreign].key,
dir);
p_rule->n_foreign++;
}
@@ -95,7 +107,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
}
}
int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int rule_num, struct maat *maat_instance, const char *dir)
int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, struct maat *maat_instance, const char *dir)
{
int rule_with_foreign_key = 0;
@@ -112,7 +125,8 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in
}
int foreign_columns[8];
int n_foreign_column = plugin_table_get_foreign_column((struct plugin_schema *)schema, foreign_columns);
int n_foreign_column = plugin_table_get_foreign_column((struct plugin_schema *)schema,
foreign_columns);
if (0 == n_foreign_column) {
continue;
}
@@ -175,9 +189,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
redisReply* reply = NULL;
for (i = 0; i < rule_num; i++) {
snprintf(redis_cmd, sizeof(redis_cmd), "GET %s:%s,%lu", mr_key_prefix[rule_list[i].op],
rule_list[i].table_name,
rule_list[i].rule_id);
snprintf(redis_cmd, sizeof(redis_cmd),
"GET %s:%s,%lu", mr_key_prefix[rule_list[i].op],
rule_list[i].table_name,
rule_list[i].rule_id);
ret = redisAppendCommand(c, redis_cmd);
assert(ret == REDIS_OK);
}
@@ -185,9 +200,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
for (i = 0; i < rule_num; i++) {
ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) {
log_error(logger, MODULE_REDIS_MONITOR, "Redis GET %s:%s,%lu failed, redis server error",
mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id);
log_error(logger, MODULE_REDIS_MONITOR,
"Redis GET %s:%s,%lu failed, redis server error",
mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id);
error_happened = 1;
break;
}
@@ -199,9 +215,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
retry_ids[failed_cnt] = i;
failed_cnt++;
} else {
log_error(logger, MODULE_REDIS_MONITOR, "Redis GET %s:%s,%lu failed",
mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id);
log_error(logger, MODULE_REDIS_MONITOR,
"Redis GET %s:%s,%lu failed",
mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id);
error_happened = 1;
}
}
@@ -218,9 +235,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
int idx = 0;
for (i = 0; i < failed_cnt; i++) {
idx = retry_ids[i];
snprintf(redis_cmd, sizeof(redis_cmd), "GET %s:%s,%lu", mr_key_prefix[MAAT_OP_DEL],
rule_list[idx].table_name,
rule_list[idx].rule_id);
snprintf(redis_cmd, sizeof(redis_cmd),
"GET %s:%s,%lu", mr_key_prefix[MAAT_OP_DEL],
rule_list[idx].table_name,
rule_list[idx].rule_id);
ret = redisAppendCommand(c, redis_cmd);
}
@@ -228,7 +246,9 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
idx = retry_ids[i];
ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) {
log_error(logger, MODULE_REDIS_MONITOR, "redis command %s failed, redis server error", redis_cmd);
log_error(logger, MODULE_REDIS_MONITOR,
"redis command %s failed, redis server error",
redis_cmd);
FREE(retry_ids);
return -1;
}
@@ -237,12 +257,14 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
rule_list[idx].table_line = maat_strdup(reply->str);
} else if(reply->type==REDIS_REPLY_ERROR) {
//Deal with Redis response: "Loading Redis is loading the database in memory"
log_error(logger, MODULE_REDIS_MONITOR, "redis command %s error, reply type=%d, error str=%s",
redis_cmd, reply->type, reply->str);
log_error(logger, MODULE_REDIS_MONITOR,
"redis command %s error, reply type=%d, error str=%s",
redis_cmd, reply->type, reply->str);
} else {
//Handle type "nil"
log_error(logger, MODULE_REDIS_MONITOR, "redis command %s failed, reply type=%d",
redis_cmd, reply->type);
log_error(logger, MODULE_REDIS_MONITOR,
"redis command %s failed, reply type=%d",
redis_cmd, reply->type);
}
freeReplyObject(reply);
@@ -254,8 +276,8 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
return 0;
}
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num,
int print_process, struct log_handle *logger)
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_process, struct log_handle *logger)
{
int max_redis_batch = 4096;
int success_cnt = 0;
@@ -286,14 +308,17 @@ int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int
}
int get_inc_key_list(long long instance_version, long long target_version,
redisContext *c, struct serial_rule **list, struct log_handle *logger)
redisContext *c, struct serial_rule **list,
struct log_handle *logger)
{
//Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version.
//The elements are considered to be ordered from low to high scores(instance_version).
redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld", mr_status_sset,
instance_version,target_version);
redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld",
mr_status_sset, instance_version,
target_version);
if (NULL == reply) {
log_error(logger, MODULE_REDIS_MONITOR, "GET %s failed with a NULL reply, error: %s",
log_error(logger, MODULE_REDIS_MONITOR,
"GET %s failed with a NULL reply, error: %s",
mr_status_sset, c->errstr);
return -1;
}
@@ -307,10 +332,14 @@ int get_inc_key_list(long long instance_version, long long target_version,
return 0;
}
redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s", mr_status_sset, reply->element[0]->str);
redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s",
mr_status_sset,
reply->element[0]->str);
if (tmp_reply->type != REDIS_REPLY_STRING) {
log_error(logger, MODULE_REDIS_MONITOR, "ZSCORE %s %s failed Version: %lld->%lld",
mr_status_sset, reply->element[0]->str, instance_version, target_version);
log_error(logger, MODULE_REDIS_MONITOR,
"ZSCORE %s %s failed Version: %lld->%lld",
mr_status_sset, reply->element[0]->str,
instance_version, target_version);
freeReplyObject(tmp_reply);
tmp_reply = NULL;
freeReplyObject(reply);
@@ -327,7 +356,8 @@ int get_inc_key_list(long long instance_version, long long target_version,
}
if (nearest_rule_version != instance_version + 1) {
log_info(logger, MODULE_REDIS_MONITOR, "Noncontinuous VERSION Redis: %lld MAAT: %lld",
log_info(logger, MODULE_REDIS_MONITOR,
"Noncontinuous VERSION Redis: %lld MAAT: %lld",
nearest_rule_version, instance_version);
}
@@ -338,10 +368,11 @@ int get_inc_key_list(long long instance_version, long long target_version,
for (i = 0, j = 0; i < (int)reply->elements; i++) {
assert(reply->element[i]->type == REDIS_REPLY_STRING);
int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu", op_str,
s_rule[j].table_name, &(s_rule[j].rule_id));
int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu",
op_str, s_rule[j].table_name, &(s_rule[j].rule_id));
if (ret != 3 || s_rule[i].rule_id < 0) {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key: %s", reply->element[i]->str);
log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key: %s", reply->element[i]->str);
continue;
}
@@ -350,7 +381,8 @@ int get_inc_key_list(long long instance_version, long long target_version,
} else if(strncmp(op_str, "DEL", strlen("DEL")) == 0) {
s_rule[j].op = MAAT_OP_DEL;
} else {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key: %s", reply->element[i]->str);
log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key: %s", reply->element[i]->str);
continue;
}
j++;
@@ -398,7 +430,8 @@ struct serial_rule *serial_rule_clone(const struct serial_rule *s_rule)
new_rule->f_keys[j].key = ALLOC(char, s_rule->f_keys[j].key_len);
memcpy(new_rule->f_keys[j].key, s_rule->f_keys[j].key, s_rule->f_keys[j].key_len);
new_rule->f_keys[j].filename = ALLOC(char, strlen(s_rule->f_keys[j].filename));
memcpy(new_rule->f_keys[j].filename, s_rule->f_keys[j].filename, strlen(s_rule->f_keys[j].filename));
memcpy(new_rule->f_keys[j].filename, s_rule->f_keys[j].filename,
strlen(s_rule->f_keys[j].filename));
}
return new_rule;
@@ -466,10 +499,10 @@ int recovery_history_version(const struct serial_rule *current, int current_num,
return ret;
}
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version,
long long *new_version, struct table_manager *tbl_mgr,
struct serial_rule **list, int *update_type, int cumulative_off,
struct log_handle *logger)
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version,
long long desired_version, long long *new_version,
struct table_manager *tbl_mgr, struct serial_rule **list,
int *update_type, int cumulative_off, struct log_handle *logger)
{
int rule_num = 0;
long long target_version = 0;
@@ -478,20 +511,23 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
redisReply *reply = (redisReply *)redisCommand(c, "GET MAAT_VERSION");
if (reply != NULL) {
if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) {
log_error(logger, MODULE_REDIS_MONITOR, "GET MAAT_VERSION failed, maybe Redis is busy");
log_error(logger, MODULE_REDIS_MONITOR,
"GET MAAT_VERSION failed, maybe Redis is busy");
freeReplyObject(reply);
reply = NULL;
return -1;
}
} else {
log_error(logger, MODULE_REDIS_MONITOR, "GET MAAT_VERSION failed with NULL reply, error: %s", c->errstr);
log_error(logger, MODULE_REDIS_MONITOR,
"GET MAAT_VERSION failed with NULL reply, error: %s", c->errstr);
return -1;
}
long long redis_version = maat_cmd_read_redis_integer(reply);
if (redis_version < 0) {
if (reply->type == REDIS_REPLY_ERROR) {
log_error(logger, MODULE_REDIS_MONITOR, "Redis Communication error: %s", reply->str);
log_error(logger, MODULE_REDIS_MONITOR,
"Redis Communication error: %s", reply->str);
}
return -1;
}
@@ -508,7 +544,8 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
}
if (redis_version < instance_version) {
log_error(logger, MODULE_REDIS_MONITOR, "VERSION roll back MAAT: %lld -> Redis: %lld",
log_error(logger, MODULE_REDIS_MONITOR,
"VERSION roll back MAAT: %lld -> Redis: %lld",
instance_version, redis_version);
goto FULL_UPDATE;
}
@@ -521,7 +558,8 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
do {
target_version++;
rule_num = get_inc_key_list(instance_version, target_version, c, &s_rule_array, logger);
rule_num = get_inc_key_list(instance_version, target_version,
c, &s_rule_array, logger);
if (rule_num > 0) {
break;
} else if (rule_num < 0) {
@@ -533,21 +571,25 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
} while (0 == rule_num && target_version <= redis_version && 1 == cumulative_off);
if (0 == rule_num) {
log_info(logger, MODULE_REDIS_MONITOR, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s",
mr_status_sset, instance_version, target_version-1, cumulative_off == 1 ? "OFF" : "ON");
log_info(logger, MODULE_REDIS_MONITOR,
"Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s",
mr_status_sset, instance_version, target_version-1,
cumulative_off == 1 ? "OFF" : "ON");
return 0;
}
log_info(logger, MODULE_REDIS_MONITOR, "Inc Update from instance_version %lld to %lld (%d entries)",
log_info(logger, MODULE_REDIS_MONITOR,
"Inc Update from instance_version %lld to %lld (%d entries)",
instance_version, target_version, rule_num);
*list = s_rule_array;
*update_type = CM_UPDATE_TYPE_INC;
*update_type = MAAT_UPDATE_TYPE_INC;
*new_version = target_version;
return rule_num;
FULL_UPDATE:
log_info(logger, MODULE_REDIS_MONITOR, "Initiate full update from instance_version %lld to %lld",
log_info(logger, MODULE_REDIS_MONITOR,
"Initiate full update from instance_version %lld to %lld",
instance_version, desired_version == 0 ? redis_version : desired_version);
size_t append_cmd_cnt = 0;
int ret = redisAppendCommand(c, "MULTI");
@@ -569,12 +611,14 @@ FULL_UPDATE:
reply = maat_cmd_wrap_redis_command(c, "EXEC");
if (NULL == reply) {
log_error(logger, MODULE_REDIS_MONITOR, "Redis Communication error: %s", c->errstr);
log_error(logger, MODULE_REDIS_MONITOR,
"Redis Communication error: %s", c->errstr);
return -1;
}
if (reply->type != REDIS_REPLY_ARRAY) {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key List type %d", reply->type);
log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key List type %d", reply->type);
freeReplyObject(reply);
reply = NULL;
return -1;
@@ -583,7 +627,8 @@ FULL_UPDATE:
*new_version = maat_cmd_read_redis_integer(reply->element[0]);
redisReply *sub_reply = reply->element[1];
if (sub_reply->type != REDIS_REPLY_ARRAY) {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key List type %d", sub_reply->type);
log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key List type %d", sub_reply->type);
freeReplyObject(reply);
reply = NULL;
return -1;
@@ -593,17 +638,21 @@ FULL_UPDATE:
s_rule_array = ALLOC(struct serial_rule, sub_reply->elements);
for (i = 0, full_idx = 0; i < sub_reply->elements; i++) {
if (sub_reply->element[i]->type != REDIS_REPLY_STRING) {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key Type: %d", sub_reply->element[i]->type);
log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key Type: %d", sub_reply->element[i]->type);
continue;
}
ret = sscanf(sub_reply->element[i]->str, "%*[^:]:%[^,],%ld",
s_rule_array[full_idx].table_name,
&(s_rule_array[full_idx].rule_id));
s_rule_array[full_idx].table_name,
&(s_rule_array[full_idx].rule_id));
s_rule_array[full_idx].op = MAAT_OP_ADD;
if (ret != 2 || s_rule_array[full_idx].rule_id < 0 || strlen(s_rule_array[full_idx].table_name) == 0) {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key Format: %s", sub_reply->element[i]->str);
if (ret != 2 || s_rule_array[full_idx].rule_id < 0 ||
strlen(s_rule_array[full_idx].table_name) == 0) {
log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key Format: %s",
sub_reply->element[i]->str);
continue;
}
@@ -623,22 +672,27 @@ FULL_UPDATE:
if (desired_version != 0) {
struct serial_rule *changed_rule_array = NULL;
int changed_rule_num = get_inc_key_list(desired_version, redis_version, c, &changed_rule_array, logger);
int changed_rule_num = get_inc_key_list(desired_version, redis_version,
c, &changed_rule_array, logger);
if (changed_rule_num < 0) {
log_error(logger, MODULE_REDIS_MONITOR, "Recover history version %lld faild where as redis version is %lld",
log_error(logger, MODULE_REDIS_MONITOR,
"Recover history version %lld faild where as redis version is %lld",
desired_version, redis_version);
} else if(0 == changed_rule_num) {
log_error(logger, MODULE_REDIS_MONITOR, "Nothing to recover from history version %lld to redis version is %lld",
log_error(logger, MODULE_REDIS_MONITOR,
"Nothing to recover from history version %lld to redis version is %lld",
desired_version, redis_version);
} else {
struct serial_rule *history_rule_array = NULL;
ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array);
ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array,
changed_rule_num, &history_rule_array);
if (ret > 0) {
FREE(s_rule_array);
s_rule_array = history_rule_array;
rule_num = ret;
*new_version = desired_version;
log_info(logger, MODULE_REDIS_MONITOR, "Successfully recovered from history version %lld to redis version is %lld",
log_info(logger, MODULE_REDIS_MONITOR,
"Successfully recovered from history version %lld to redis version is %lld",
desired_version, redis_version);
}
}
@@ -646,14 +700,15 @@ FULL_UPDATE:
}
*list = s_rule_array;
*update_type = CM_UPDATE_TYPE_FULL;
log_info(logger, MODULE_REDIS_MONITOR, "Full update %d keys of version %lld", rule_num, *new_version);
*update_type = MAAT_UPDATE_TYPE_FULL;
log_info(logger, MODULE_REDIS_MONITOR,
"Full update %d keys of version %lld", rule_num, *new_version);
return rule_num ;
}
void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn,
struct log_handle *logger)
void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_fn, struct log_handle *logger)
{
int i = 0;
int j = 0;
@@ -676,8 +731,9 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
ret = remove(rule_list[i].f_keys[j].filename);
if (ret == -1) {
log_error(logger, MODULE_REDIS_MONITOR, "Foreign content file %s remove failed",
rule_list[i].f_keys[j].filename);
log_error(logger, MODULE_REDIS_MONITOR,
"Foreign content file %s remove failed",
rule_list[i].f_keys[j].filename);
}
}
} else {
@@ -734,7 +790,8 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
fclose(fp);
fp = NULL;
if (1 == print_fn) {
printf("Written foreign content %s\n", s_rule->f_keys[track[i].foreign_idx].filename);
printf("Written foreign content %s\n",
s_rule->f_keys[track[i].foreign_idx].filename);
}
}
}
@@ -747,8 +804,8 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
return;
}
void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn,
struct log_handle *logger)
void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_fn, struct log_handle *logger)
{
int max_redis_batch = 4096;
int success_cnt = 0;
@@ -797,12 +854,14 @@ void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule)
pos_rewrite_line += strlen(s_rule->f_keys[i].filename);
}
strncat(pos_rewrite_line, pos_origin_line, strlen(s_rule->table_line) - (pos_origin_line - s_rule->table_line));
strncat(pos_rewrite_line, pos_origin_line,
strlen(s_rule->table_line) - (pos_origin_line - s_rule->table_line));
FREE(s_rule->table_line);
s_rule->table_line = rewrite_line;
}
void expected_reply_add(struct expected_reply* expected, int s_rule_seq, int type, long long integer)
void expected_reply_add(struct expected_reply* expected, int s_rule_seq,
int type, long long integer)
{
int i = expected->possible_reply_num;
assert(i < POSSIBLE_REDIS_REPLY_SIZE);
@@ -816,7 +875,8 @@ int redlock_try_lock(redisContext *c, const char *lock_name, long long expire)
{
int ret = 0;
redisReply *reply = maat_cmd_wrap_redis_command(c, "SET %s locked NX PX %lld", lock_name, expire);
redisReply *reply = maat_cmd_wrap_redis_command(c, "SET %s locked NX PX %lld",
lock_name, expire);
if (reply->type == REDIS_REPLY_NIL) {
ret = 0;
} else {
@@ -829,8 +889,9 @@ int redlock_try_lock(redisContext *c, const char *lock_name, long long expire)
return ret;
}
long long exec_serial_rule_begin(redisContext* c, size_t rule_num, size_t renew_rule_num,
int *renew_allowed, long long *transaction_version)
long long exec_serial_rule_begin(redisContext* c, size_t rule_num,
size_t renew_rule_num, int *renew_allowed,
long long *transaction_version)
{
int ret = -1;
redisReply *data_reply = NULL;
@@ -878,8 +939,9 @@ const char* lua_exec_done=
"redis.call(\'del\', KEYS[4]);"
"redis.call(\'zadd\', KEYS[3], ARGV[1], maat_version);"
"return maat_version;";
redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list, long long server_time,
int renew_allowed, struct expected_reply *expect_reply, size_t *cnt)
redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list,
long long server_time, int renew_allowed,
struct expected_reply *expect_reply, size_t *cnt)
{
redisReply *data_reply = NULL;
@@ -907,8 +969,10 @@ redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list,
return data_reply;
}
void exec_serial_rule(redisContext *c, const char *transaction_list, struct serial_rule *s_rule, size_t rule_num,
struct expected_reply *expect_reply, size_t *cnt, size_t offset, int renew_allowed)
void exec_serial_rule(redisContext *c, const char *transaction_list,
struct serial_rule *s_rule, size_t rule_num,
struct expected_reply *expect_reply, size_t *cnt,
size_t offset, int renew_allowed)
{
size_t i = 0;
size_t append_cmd_cnt = 0;
@@ -1053,8 +1117,9 @@ int mr_operation_success(redisReply *actual_reply, struct expected_reply *expect
return 0;
}
int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num,
long long server_time, struct log_handle *logger)
int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
size_t serial_rule_num, long long server_time,
struct log_handle *logger)
{
size_t i = 0;
size_t rule_seq = 0;
@@ -1079,7 +1144,8 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri
}
}
int ret = exec_serial_rule_begin(c, serial_rule_num, renew_num, &renew_allowed, &transaction_version);
int ret = exec_serial_rule_begin(c, serial_rule_num, renew_num, &renew_allowed,
&transaction_version);
//Preconditions for transaction are not satisfied.
if (ret != 0) {
success_cnt = -1;
@@ -1087,18 +1153,20 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri
}
if (transaction_version > 0) {
snprintf(transaction_list, sizeof(transaction_list), "MAAT_TRANSACTION_%lld", transaction_version);
snprintf(transaction_list, sizeof(transaction_list), "MAAT_TRANSACTION_%lld",
transaction_version);
}
while (success_cnt < serial_rule_num) {
size_t batch_cnt = MIN(serial_rule_num - success_cnt, max_redis_batch);
exec_serial_rule(c, transaction_list, s_rule + success_cnt, batch_cnt, expected_reply, &multi_cmd_cnt,
success_cnt, renew_allowed);
exec_serial_rule(c, transaction_list, s_rule + success_cnt, batch_cnt,
expected_reply, &multi_cmd_cnt, success_cnt, renew_allowed);
assert(multi_cmd_cnt<max_multi_cmd_num);
success_cnt+=batch_cnt;
}
transaction_reply = exec_serial_rule_end(c, transaction_list, server_time, renew_allowed, expected_reply, &multi_cmd_cnt);
transaction_reply = exec_serial_rule_end(c, transaction_list, server_time, renew_allowed,
expected_reply, &multi_cmd_cnt);
if (1 == mr_transaction_success(transaction_reply)) {
assert(transaction_reply->elements == multi_cmd_cnt);
for (i = 0; i < multi_cmd_cnt; i++) {
@@ -1138,8 +1206,10 @@ error_out:
if (renew_num > 0 && renew_allowed != 1) {
for (i = 0; i < (unsigned int)serial_rule_num; i++) {
if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) {
log_error(logger, MODULE_REDIS_MONITOR, "%s %s %lu is not allowed due to lock contention",
mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, s_rule[i].rule_id);
log_error(logger, MODULE_REDIS_MONITOR,
"%s %s %lu is not allowed due to lock contention",
mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name,
s_rule[i].rule_id);
}
}
@@ -1171,11 +1241,11 @@ void cleanup_update_status(redisContext *c, struct log_handle *logger)
int append_cmd_cnt = 0;
redisAppendCommand(c, "ZRANGEBYSCORE %s -inf %lld",
mr_version_sset, server_time - MAAT_REDIS_SYNC_TIME);
mr_version_sset, server_time - MAAT_REDIS_SYNC_TIME);
append_cmd_cnt++;
redisAppendCommand(c, "ZREMRANGEBYSCORE %s -inf %lld",
mr_version_sset,server_time - MAAT_REDIS_SYNC_TIME);
mr_version_sset,server_time - MAAT_REDIS_SYNC_TIME);
append_cmd_cnt++;
//consume reply "OK" and "QUEUED".
@@ -1207,8 +1277,9 @@ void cleanup_update_status(redisContext *c, struct log_handle *logger)
reply = NULL;
//To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally.
reply = maat_cmd_wrap_redis_command(c, "ZREMRANGEBYSCORE %s %lld %lld", mr_status_sset,
version_lower_bound, version_upper_bound);
reply = maat_cmd_wrap_redis_command(c, "ZREMRANGEBYSCORE %s %lld %lld",
mr_status_sset, version_lower_bound,
version_upper_bound);
entry_num = maat_cmd_read_redis_integer(reply);
freeReplyObject(reply);
reply = NULL;
@@ -1232,7 +1303,8 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger)
return;
}
redisReply *data_reply= maat_cmd_wrap_redis_command(c, "ZRANGEBYSCORE %s -inf %lld", mr_expire_sset, server_time);
redisReply *data_reply= maat_cmd_wrap_redis_command(c, "ZRANGEBYSCORE %s -inf %lld",
mr_expire_sset, server_time);
if (data_reply->type != REDIS_REPLY_ARRAY || 0 == data_reply->elements) {
freeReplyObject(data_reply);
data_reply = NULL;
@@ -1244,7 +1316,8 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger)
for (size_t i = 0; i < s_rule_num; i++) {
s_rule[i].op = MAAT_OP_DEL;
ret = sscanf(data_reply->element[i]->str, "%[^,],%ld", s_rule[i].table_name, &(s_rule[i].rule_id));
ret = sscanf(data_reply->element[i]->str, "%[^,],%ld",
s_rule[i].table_name, &(s_rule[i].rule_id));
assert(ret == 2);
}
freeReplyObject(data_reply);
@@ -1254,9 +1327,11 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger)
if (success_cnt < 0) {
log_error(logger, MODULE_REDIS_MONITOR, "maat_cmd_write_rule failed.");
} else if (success_cnt == (int)s_rule_num) {
log_info(logger, MODULE_REDIS_MONITOR, "Succesfully expired %zu rules in Redis", s_rule_num);
log_info(logger, MODULE_REDIS_MONITOR,
"Succesfully expired %zu rules in Redis", s_rule_num);
} else {
log_error(logger, MODULE_REDIS_MONITOR, "Failed to expired %d of %zu rules in Redis, try later",
log_error(logger, MODULE_REDIS_MONITOR,
"Failed to expired %d of %zu rules in Redis, try later",
s_rule_num - success_cnt, s_rule_num);
}
@@ -1301,7 +1376,9 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
}
log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Reconnecting...");
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db,
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip,
mr_ctx->redis_port,
mr_ctx->redis_db,
maat_instance->logger);
if (NULL == mr_ctx->read_ctx) {
return;
@@ -1312,11 +1389,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
struct serial_rule *rule_list = NULL;
long long new_version = 0;
int update_type = CM_UPDATE_TYPE_INC;
int update_type = MAAT_UPDATE_TYPE_INC;
int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, maat_instance->load_specific_version,
&new_version, maat_instance->tbl_mgr, &rule_list,
&update_type, maat_instance->cumulative_update_off,
int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version,
maat_instance->load_specific_version,
&new_version, maat_instance->tbl_mgr,
&rule_list, &update_type,
maat_instance->cumulative_update_off,
maat_instance->logger);
//redis communication error
if (rule_num < 0) {
@@ -1327,12 +1406,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
maat_instance->load_specific_version = 0;//only valid for one time.
//error or nothing changed
if (0 == rule_num && update_type == CM_UPDATE_TYPE_INC) {
if (0 == rule_num && update_type == MAAT_UPDATE_TYPE_INC) {
return;
}
if (rule_num > 0) {
ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_instance->logger);
ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num,
0, maat_instance->logger);
//redis communication error
if (ret < 0) {
redisFree(mr_ctx->read_ctx);
@@ -1354,15 +1434,19 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
goto clean_up;
}
ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num, maat_instance, maat_instance->foreign_cont_dir);
ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num,
maat_instance, maat_instance->foreign_cont_dir);
if (ret > 0) {
maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_instance->logger);
maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0,
maat_instance->logger);
}
}
start_fn(new_version, update_type, u_param);
log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Start %s update: %lld -> %lld (%d entries)",
update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num);
log_info(maat_instance->logger, MODULE_REDIS_MONITOR,
"Start %s update: %lld -> %lld (%d entries)",
update_type == MAAT_UPDATE_TYPE_INC ? "INC" : "FULL",
version, new_version, rule_num);
for (i = 0; i < rule_num; i++) {
if (NULL == rule_list[i].table_line) {
@@ -1385,7 +1469,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
ret = invalidate_line(rule_list[i].table_line, table_type, valid_column);
if (ret < 0) {
log_error(maat_instance->logger, MODULE_REDIS_MONITOR,
"Invalidate line failed, invaid format %s", rule_list[i].table_line);
"Invalidate line failed, invaid format %s",
rule_list[i].table_line);
continue;
}
}

81
src/maat_rule.cpp
View File

@@ -17,7 +17,6 @@
#include <sys/stat.h>
#include <assert.h>
#include "utils.h"
#include "json2iris.h"
#include "log/log.h"
#include "cJSON/cJSON.h"
@@ -57,18 +56,18 @@ void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *)
static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, size_t n_accept_tag)
{
if (NULL == tag_obj || NULL == accept_tags) {
return -1;
return TAG_MATCH_ERR;
}
cJSON *tab_name_obj = cJSON_GetObjectItem(tag_obj, "tag");
if (NULL == tab_name_obj || tab_name_obj->type != cJSON_String) {
return -1;
return TAG_MATCH_ERR;
}
const char *tag_name = tab_name_obj->valuestring;
cJSON *tag_vals_array = cJSON_GetObjectItem(tag_obj, "value");
if (NULL == tag_vals_array || tag_vals_array->type != cJSON_Array) {
return -1;
return TAG_MATCH_ERR;
}
int name_matched = 0;
@@ -82,7 +81,7 @@ static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags,
for (int j = 0; j < n_val; j++) {
cJSON *tag_val_obj = cJSON_GetArrayItem(tag_vals_array, j);
if (NULL == tag_val_obj || tag_val_obj->type != cJSON_String) {
return -1;
return TAG_MATCH_ERR;
}
const char *tag_val = tag_val_obj->valuestring;
@@ -95,16 +94,16 @@ static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags,
//make sure the overlap is ended with a '/'
if (0 == strncmp(accept_tags[i].tag_val, tag_val, strlen(tag_val)) &&
(strlen(accept_tags[i].tag_val) == strlen(tag_val) || accept_tags[i].tag_val[strlen(tag_val)] == '/')) {
return 1;
return TAG_MATCH_MATCHED;
}
}
}
//no matched name is considered as a
if (name_matched > 0) {
return 0;
return TAG_MATCH_UNMATCHED;
} else {
return 1;
return TAG_MATCH_MATCHED;
}
}
@@ -117,12 +116,12 @@ static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_ta
for (int i = 0; i < n_tag; i++) {
cJSON *tag_obj = cJSON_GetArrayItem(tag_set, i);
if (NULL == tag_obj || tag_obj->type != cJSON_Object) {
goto error;
return TAG_MATCH_ERR;
}
int ret = compare_each_tag(tag_obj, accept_tags, n_accept_tag);
if (ret < 0) {
return -1;
return TAG_MATCH_ERR;
}
if(1 == ret) {
@@ -131,13 +130,10 @@ static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_ta
}
if (matched == n_tag) {
return 1;
return TAG_MATCH_MATCHED;
} else {
return 0;
return TAG_MATCH_UNMATCHED;
}
error:
return -1;
}
//@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]}
@@ -176,7 +172,7 @@ size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_
//@return 1 on match, 0 on not match, -1 on error.
int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag)
{
int ret = -1;
int ret = TAG_MATCH_ERR;
int n_set = 0;
cJSON *tag_set = NULL;
cJSON *tag_set_array = NULL;
@@ -200,7 +196,7 @@ int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, si
ret = compare_each_tag_set(tag_set, accept_tags, n_accept_tag);
//match or error occurs.
if (ret != 0) {
if (ret != TAG_MATCH_UNMATCHED) {
break;
}
}
@@ -280,25 +276,11 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt)
FREE(maat_rt);
}
int maat_runtime_updating_flag(struct maat_runtime *maat_rt)
{
int flag = -1;
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
flag = table_manager_runtime_updating_flag(maat_rt->ref_tbl_mgr, i);
if (1 == flag) {
return 1;
}
}
return 0;
}
void maat_start_cb(long long new_version, int update_type, void *u_param)
{
struct maat *maat_instance = (struct maat *)u_param;
if (update_type == CM_UPDATE_TYPE_FULL) {
if (update_type == MAAT_UPDATE_TYPE_FULL) {
maat_instance->creating_maat_rt = maat_runtime_create(new_version, maat_instance);
} else {
maat_instance->maat_version = new_version;
@@ -390,14 +372,18 @@ void maat_finish_cb(void *u_param)
if (maat_instance->creating_maat_rt != NULL) {
maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt);
maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger);
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Full config version %llu load %d entries complete\n",
maat_instance->creating_maat_rt->version, maat_instance->creating_maat_rt->rule_num);
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Full config version %llu load %d entries complete",
maat_instance->creating_maat_rt->version,
maat_instance->creating_maat_rt->rule_num);
} else if (maat_instance->maat_rt != NULL) {
maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt);
maat_instance->maat_rt->version = maat_instance->maat_version;
maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger);
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Inc config version %llu load %d entries complete\n",
maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num);
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Inc config version %llu load %d entries complete",
maat_instance->maat_rt->version,
maat_instance->maat_rt->rule_num);
}
}
@@ -420,7 +406,8 @@ void *rule_monitor_loop(void *arg)
pthread_mutex_lock(&(maat_instance->background_update_mutex));
/* if deferred load on */
if (maat_instance->deferred_load != 0) {
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Deferred Loading ON, updating in %s", __func__);
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Deferred Loading ON, updating in %s", __func__);
maat_read_full_config(maat_instance);
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
@@ -478,13 +465,16 @@ void *rule_monitor_loop(void *arg)
if (old_maat_rt != NULL) {
if (maat_instance->maat_rt->version > old_maat_rt->version) {
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Maat version updated %lld -> %lld\n",
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Maat version updated %lld -> %lld\n",
old_maat_rt->version, maat_instance->maat_rt->version);
} else {
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Maat version roll back %lld -> %lld\n",
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Maat version roll back %lld -> %lld\n",
old_maat_rt->version, maat_instance->maat_rt->version);
}
maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy);
maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt,
(void (*)(void*))maat_runtime_destroy);
}
maat_instance->creating_maat_rt = NULL;
@@ -493,12 +483,11 @@ void *rule_monitor_loop(void *arg)
}
if (maat_instance->maat_rt != NULL) {
int updating_flag = maat_runtime_updating_flag(maat_instance->maat_rt);
time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time;
if ((updating_flag > 0) && (time_window >= maat_instance->rule_effect_interval_ms / 1000)) {
if (time_window >= maat_instance->rule_effect_interval_ms / 1000) {
maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger);
log_info(maat_instance->logger,MODULE_MAAT_RULE,
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Actual update config version %u, %d entries load to rulescan after postpone.",
maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num);
}
@@ -514,6 +503,12 @@ void *rule_monitor_loop(void *arg)
maat_garbage_bin_free(maat_instance->garbage_bin);
table_manager_destroy(maat_instance->tbl_mgr);
alignment_int64_array_free(maat_instance->outer_state_cnt);
alignment_int64_array_free(maat_instance->compile_state_cnt);
alignment_int64_array_free(maat_instance->thread_call_cnt);
alignment_int64_array_free(maat_instance->hit_cnt);
alignment_int64_array_free(maat_instance->not_grp_hit_cnt);
if (maat_instance->input_mode == DATA_SOURCE_REDIS) {
if (maat_instance->mr_ctx.read_ctx != NULL) {
redisFree(maat_instance->mr_ctx.read_ctx);

174
src/maat_table.cpp
View File

@@ -12,7 +12,6 @@
#include <assert.h>
#include "log/log.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_table.h"
#include "maat_rule.h"
@@ -26,7 +25,7 @@
#include "maat_ip_plugin.h"
#include "maat_virtual.h"
#define MODULE_TABLE module_name_str("maat.table")
#define MODULE_TABLE module_name_str("maat.table")
struct table_item {
enum table_type table_type;
@@ -50,21 +49,28 @@ struct table_manager {
size_t n_accept_tag;
int default_compile_table_id;
int g2g_table_id;
struct maat_kv_store *tablename2id_map;
struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
};
struct table_operations {
enum table_type type;
void *(*new_schema)(cJSON *json, const char *table_name, struct log_handle *logger);
void *(*new_schema)(cJSON *json, struct table_manager *tbl_mgr, const char *table_name,
struct log_handle *logger);
void (*free_schema)(void *schema);
void *(*new_runtime)(void *schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
void *(*new_runtime)(void *schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void (*free_runtime)(void *runtime);
int (*update_runtime)(void *runtime, void *schema, const char *line, int valid_column);
int (*commit_runtime)(void *runtime);
int (*runtime_updating_flag)(void *runtime);
};
struct table_operations table_ops[TABLE_TYPE_MAX] = {
@@ -75,8 +81,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = expr_runtime_new,
.free_runtime = expr_runtime_free,
.update_runtime = expr_runtime_update,
.commit_runtime = expr_runtime_commit,
.runtime_updating_flag = expr_runtime_updating_flag
.commit_runtime = expr_runtime_commit
},
{
.type = TABLE_TYPE_EXPR_PLUS,
@@ -85,18 +90,16 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = expr_runtime_new,
.free_runtime = expr_runtime_free,
.update_runtime = expr_runtime_update,
.commit_runtime = expr_runtime_commit,
.runtime_updating_flag = expr_runtime_updating_flag
.commit_runtime = expr_runtime_commit
},
{
.type = TABLE_TYPE_IP_PLUS,
.new_schema = ip_plus_schema_new,
.free_schema = ip_plus_schema_free,
.new_runtime = ip_plus_runtime_new,
.free_runtime = ip_plus_runtime_free,
.update_runtime = ip_plus_runtime_update,
.commit_runtime = ip_plus_runtime_commit,
.runtime_updating_flag = ip_plus_runtime_updating_flag
.new_schema = ip_schema_new,
.free_schema = ip_schema_free,
.new_runtime = ip_runtime_new,
.free_runtime = ip_runtime_free,
.update_runtime = ip_runtime_update,
.commit_runtime = ip_runtime_commit
},
{
.type = TABLE_TYPE_INTERVAL,
@@ -105,8 +108,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL,
.runtime_updating_flag = NULL
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_INTERVAL_PLUS,
@@ -115,8 +117,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL,
.runtime_updating_flag = NULL
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_DIGEST,
@@ -125,8 +126,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL,
.runtime_updating_flag = NULL
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_SIMILARITY,
@@ -135,8 +135,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL,
.runtime_updating_flag = NULL
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_CONJUNCTION,
@@ -145,8 +144,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL,
.runtime_updating_flag = NULL
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_PLUGIN,
@@ -155,8 +153,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = plugin_runtime_new,
.free_runtime = plugin_runtime_free,
.update_runtime = plugin_runtime_update,
.commit_runtime = plugin_runtime_commit,
.runtime_updating_flag = plugin_runtime_updating_flag
.commit_runtime = plugin_runtime_commit
},
{
.type = TABLE_TYPE_IP_PLUGIN,
@@ -165,8 +162,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = ip_plugin_runtime_new,
.free_runtime = ip_plugin_runtime_free,
.update_runtime = ip_plugin_runtime_update,
.commit_runtime = ip_plugin_runtime_commit,
.runtime_updating_flag = ip_plugin_runtime_updating_flag
.commit_runtime = ip_plugin_runtime_commit
},
{
.type = TABLE_TYPE_FQDN_PLUGIN,
@@ -202,8 +198,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = compile_runtime_new,
.free_runtime = compile_runtime_free,
.update_runtime = compile_runtime_update,
.commit_runtime = compile_runtime_commit,
.runtime_updating_flag = NULL
.commit_runtime = compile_runtime_commit
},
{
.type = TABLE_TYPE_GROUP2GROUP,
@@ -212,8 +207,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = group2group_runtime_new,
.free_runtime = group2group_runtime_free,
.update_runtime = group2group_runtime_update,
.commit_runtime = group2group_runtime_commit,
.runtime_updating_flag = NULL
.commit_runtime = group2group_runtime_commit
},
{
.type = TABLE_TYPE_GROUP2COMPILE,
@@ -222,18 +216,19 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = group2compile_runtime_new,
.free_runtime = group2compile_runtime_free,
.update_runtime = group2compile_runtime_update,
.commit_runtime = NULL,
.runtime_updating_flag = NULL
.commit_runtime = NULL
}
};
void *maat_table_schema_new(cJSON *json, const char *table_name, enum table_type table_type,
void *maat_table_schema_new(cJSON *json, const char *table_name,
enum table_type table_type,
struct table_manager *tbl_mgr,
struct log_handle *logger)
{
void *schema = NULL;
if (table_ops[table_type].new_schema != NULL) {
schema = table_ops[table_type].new_schema(json, table_name, logger);
schema = table_ops[table_type].new_schema(json, tbl_mgr, table_name, logger);
}
return schema;
@@ -378,8 +373,9 @@ void maat_table_free(struct maat_table *maat_tbl)
FREE(maat_tbl);
}
struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags,
struct log_handle *logger)
struct table_manager *
table_manager_create(const char *table_info_path, const char *accept_tags,
struct maat_garbage_bin *garbage_bin, struct log_handle *logger)
{
if (NULL == table_info_path) {
return NULL;
@@ -389,7 +385,8 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
size_t json_buff_sz = 0;
int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz);
if (ret < 0) {
log_error(logger, MODULE_TABLE, "Maat read table info %s error.", table_info_path);
log_error(logger, MODULE_TABLE,
"Maat read table info %s error.", table_info_path);
return NULL;
}
@@ -397,14 +394,16 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
cJSON *json = NULL;
root = cJSON_Parse((const char *)json_buff);
if (!root) {
log_error(logger, MODULE_TABLE, "Error before: %-200.200s", cJSON_GetErrorPtr());
log_error(logger, MODULE_TABLE,
"Error before: %-200.200s", cJSON_GetErrorPtr());
FREE(json_buff);
return NULL;
}
int json_array_size = cJSON_GetArraySize(root);
if (json_array_size <= 0) {
log_error(logger, MODULE_TABLE, "invalid json content in %s", table_info_path);
log_error(logger, MODULE_TABLE,
"invalid json content in %s", table_info_path);
free(json_buff);
return NULL;
}
@@ -416,8 +415,11 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger);
tbl_mgr->logger = logger;
tbl_mgr->tablename2id_map = maat_kv_store_new();
tbl_mgr->district_map = maat_kv_store_new();
tbl_mgr->ref_garbage_bin = garbage_bin;
int default_compile_table_id = MAX_TABLE_NUM;
int g2g_table_id = MAX_TABLE_NUM;
for (int i = 0; i < json_array_size; i++) {
json = cJSON_GetArrayItem(root, i);
@@ -437,10 +439,11 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
continue;
}
maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type, logger);
maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name,
maat_tbl->table_type, tbl_mgr, logger);
if (NULL == maat_tbl->schema) {
log_error(logger, MODULE_TABLE, "Maat table schema new failed, table_name:%s",
maat_tbl->table_name);
log_error(logger, MODULE_TABLE,
"Maat table schema new failed, table_name:%s", maat_tbl->table_name);
maat_table_free(maat_tbl);
continue;
}
@@ -451,14 +454,23 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
}
}
if (maat_tbl->table_type == TABLE_TYPE_GROUP2GROUP) {
g2g_table_id = maat_tbl->table_id;
}
tbl_mgr->tbl[maat_tbl->table_id] = maat_tbl;
tbl_mgr->n_table++;
}
}
assert(default_compile_table_id != MAX_TABLE_NUM);
assert(g2g_table_id != MAX_TABLE_NUM);
tbl_mgr->default_compile_table_id = default_compile_table_id;
tbl_mgr->g2g_table_id = g2g_table_id;
log_info(logger, MODULE_TABLE, "default compile table id: %d", default_compile_table_id);
log_info(logger, MODULE_TABLE, "group2group table id: %d", g2g_table_id);
maat_kv_store_free(reserved_word_map);
cJSON_Delete(root);
@@ -490,7 +502,6 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n
assert(tbl_mgr->n_table != 0);
size_t i = 0;
int g2g_group_id = MAX_TABLE_NUM;
enum table_type table_type = TABLE_TYPE_MAX;
for (i = 0; i < MAX_TABLE_NUM; i++) {
@@ -500,17 +511,11 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n
}
table_type = table_manager_get_table_type(tbl_mgr, i);
if (table_type == TABLE_TYPE_GROUP2GROUP) {
g2g_group_id = i;
}
assert(NULL == tbl_mgr->tbl[i]->runtime);
tbl_mgr->tbl[i]->runtime = maat_table_runtime_new(schema, table_type, max_thread_num,
garbage_bin, tbl_mgr->logger);
}
assert(g2g_group_id != MAX_TABLE_NUM);
/* group2compile runtime depends on associated compile runtime,
must make sure associated compile runtime already exist */
for (i = 0; i < MAX_TABLE_NUM; i++) {
@@ -528,10 +533,14 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n
void *schema = table_manager_get_schema(tbl_mgr, i);
int associated_compile_table_id = group2compile_associated_compile_table_id(schema);
void *compile_rt = table_manager_get_runtime(tbl_mgr, associated_compile_table_id);
int g2g_group_id = table_manager_get_group2group_table_id(tbl_mgr);
void *g2g_rt = table_manager_get_runtime(tbl_mgr, g2g_group_id);
group2compile_runtime_init(runtime, compile_rt, g2g_rt);
}
/* new district map */
tbl_mgr->district_map = maat_kv_store_new();
return 0;
}
@@ -551,6 +560,9 @@ void table_manager_runtime_destroy(struct table_manager *tbl_mgr)
maat_table_runtime_free(runtime, table_type);
tbl_mgr->tbl[i]->runtime = NULL;
}
/* free district map */
maat_kv_store_free(tbl_mgr->district_map);
}
void table_manager_destroy(struct table_manager *tbl_mgr)
@@ -580,6 +592,7 @@ void table_manager_destroy(struct table_manager *tbl_mgr)
FREE(tbl_mgr->accept_tags);
maat_kv_store_free(tbl_mgr->tablename2id_map);
FREE(tbl_mgr);
}
@@ -622,6 +635,11 @@ int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr)
return tbl_mgr->default_compile_table_id;
}
int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr)
{
return tbl_mgr->g2g_table_id;
}
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id)
{
if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) {
@@ -635,7 +653,8 @@ void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id)
return tbl_mgr->tbl[table_id]->schema;
}
struct ex_data_schema *table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id)
struct ex_data_schema *
table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id)
{
return NULL;
}
@@ -653,9 +672,21 @@ int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id)
return tbl_mgr->tbl[table_id]->valid_column;
}
size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr)
{
return tbl_mgr->n_accept_tag;
}
int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags)
{
return 0;
return compare_accept_tag(tags, tbl_mgr->accept_tags, tbl_mgr->n_accept_tag);
}
int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str,
size_t district_str_len, int *district_id)
{
return maat_kv_read_unNull(tbl_mgr->district_map,
district_str, district_str_len, district_id);
}
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id)
@@ -671,26 +702,8 @@ void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id)
return tbl_mgr->tbl[table_id]->runtime;
}
int table_manager_runtime_updating_flag(struct table_manager *tbl_mgr, int table_id)
{
if (NULL == tbl_mgr) {
return -1;
}
if (NULL == tbl_mgr->tbl[table_id]) {
return -1;
}
enum table_type table_type = tbl_mgr->tbl[table_id]->table_type;
void *runtime = table_manager_get_runtime(tbl_mgr, table_id);
if (table_ops[table_type].runtime_updating_flag != NULL) {
return table_ops[table_type].runtime_updating_flag(runtime);
}
return -1;
}
int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line)
int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id,
const char *line)
{
void *schema = table_manager_get_schema(tbl_mgr, table_id);
if (NULL == schema) {
@@ -716,7 +729,16 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, co
return -1;
}
return table_ops[table_type].update_runtime(runtime, schema, line, valid_column);
int ret = table_ops[table_type].update_runtime(runtime, schema, line, valid_column);
if (tbl_mgr->tmp_district_map != NULL) {
struct maat_kv_store *tmp_map = tbl_mgr->district_map;
tbl_mgr->district_map = tbl_mgr->tmp_district_map;
tbl_mgr->tmp_district_map = NULL;
maat_garbage_bagging(tbl_mgr->ref_garbage_bin, tmp_map, (void (*)(void *))maat_kv_store_free);
}
return ret;
}
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)

11
src/maat_utils.cpp
View File

@@ -17,7 +17,6 @@
#include <openssl/md5.h>
#include <openssl/evp.h>
#include "utils.h"
#include "maat_utils.h"
pid_t gettid()
@@ -257,8 +256,8 @@ char *md5_file(const char *filename, char *md5string)
return md5string;
}
int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, size_t *out_sz,
const char *key, const char *algorithm, int do_encrypt,
int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out,
size_t *out_sz, const char *key, const char *algorithm, int do_encrypt,
char *err_str, size_t err_str_sz)
{
OpenSSL_add_all_algorithms();
@@ -325,7 +324,8 @@ error_out:
}
int decrypt_open(const char* file_name, const char* key, const char* algorithm,
unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz)
unsigned char**pp_out, size_t *out_sz,
char* err_str, size_t err_str_sz)
{
size_t file_sz = 0;
unsigned char *file_buff = NULL;
@@ -334,7 +334,8 @@ int decrypt_open(const char* file_name, const char* key, const char* algorithm,
return -1;
}
ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz);
ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm,
0, err_str, err_str_sz);
FREE(file_buff);
return ret;

11
src/maat_virtual.cpp
View File

@@ -10,7 +10,6 @@
#include "cJSON/cJSON.h"
#include "maat_kv.h"
#include "utils.h"
#include "maat_utils.h"
#include "log/log.h"
#include "maat_virtual.h"
@@ -19,19 +18,25 @@
struct virtual_schema {
int physical_table_id[SCAN_TYPE_MAX];
int table_id;
struct table_manager *tbl_mgr;
};
void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
//size_t read_cnt = 0;
cJSON *item = cJSON_GetObjectItem(json, "physical_table");
if (NULL == item || item->type != cJSON_Array) {
log_error(logger, MODULE_VIRTUAL, "virtual table %s has no physical_table column", table_name);
log_error(logger, MODULE_VIRTUAL,
"virtual table %s has no physical_table column", table_name);
return NULL;
}
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
vt_schema->tbl_mgr = tbl_mgr;
return vt_schema;
#if 0
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);

16
src/rcu_hash.cpp
View File

@@ -14,7 +14,7 @@
#include <sys/queue.h>
#include "rcu_hash.h"
#include "utils.h"
#include "maat_utils.h"
struct rcu_hash_garbage_bag {
void *garbage;
@@ -68,7 +68,8 @@ size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable)
return htable->garbage_q_len;
}
void rcu_hash_garbage_bagging(struct rcu_hash_garbage_q* garbage_q, void* garbage, void (* func)(void *))
void rcu_hash_garbage_bagging(struct rcu_hash_garbage_q* garbage_q,
void* garbage, void (* func)(void *))
{
struct rcu_hash_garbage_bag *bag = ALLOC(struct rcu_hash_garbage_bag, 1);
@@ -171,7 +172,8 @@ void rcu_hash_commit_prepare(struct rcu_hash_table *htable)
htable->is_updating = 1;
}
void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len, void *data)
void rcu_hash_add(struct rcu_hash_table *htable, const char *key,
size_t key_len, void *data)
{
if (NULL == htable || NULL == key || 0 == key_len) {
return;
@@ -227,7 +229,8 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len
}
if (node != NULL) {
rcu_hash_garbage_bagging(&(htable->garbage_q), node, (void (*)(void*))rcu_hash_node_free);
rcu_hash_garbage_bagging(&(htable->garbage_q), node,
(void (*)(void*))rcu_hash_node_free);
htable->garbage_q_len++;
}
}
@@ -344,9 +347,4 @@ size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_a
}
return node_cnt;
}
int rcu_hash_updating_flag(struct rcu_hash_table *htable)
{
return htable->is_updating;
}

183
test/maat_framework_gtest.cpp
View File

@@ -1,6 +1,5 @@
#include <gtest/gtest.h>
#include "utils.h"
#include "maat/maat.h"
#include "maat_rule.h"
#include "maat_utils.h"
@@ -9,64 +8,95 @@
#include "json2iris.h"
#include "maat_config_monitor.h"
struct maat *g_maat_instance = NULL;
#define MODULE_FRAMEWORK_GTEST module_name_str("maat.framework_gtest")
const char *table_info_path = "./table_info.conf";
const char *json_path="./maat_json.json";
const char *json_filename = "maat_json.json";
struct maat *g_maat_instance = NULL;
TEST(maat_scan_string, hit_one_expr) {
int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL");
class MaatStringScan : public testing::Test
{
protected:
static void SetUpTestCase() {
table_id = maat_table_get_id(g_maat_instance, "HTTP_URL");
}
static void TearDownTestCase() {
}
static int table_id;
};
int MaatStringScan::table_id;
TEST_F(MaatStringScan, hit_one_expr) {
int table_id = MaatStringScan::table_id;
char scan_data[128] = "hello";
int results[5] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_HIT);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191);
struct maat_hit_path hit_path[128] = {0};
int n_read = 0;
n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, sizeof(hit_path));
EXPECT_NE(n_read, 0);
maat_state_free(&state);
}
TEST(maat_scan_string, hit_two_expr) {
int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL");
TEST_F(MaatStringScan, hit_two_expr) {
int table_id = MaatStringScan::table_id;
char data[128] = "should hit aaa bbb";
int results[5] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 2);
EXPECT_EQ(results[0], 28);
EXPECT_EQ(results[1], 27);
EXPECT_EQ(results[0], 139);
EXPECT_EQ(results[1], 138);
maat_state_free(&state);
}
TEST(maat_scan_string, hit_three_expr) {
int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL");
TEST_F(MaatStringScan, hit_three_expr) {
int table_id = MaatStringScan::table_id;
char data[128] = "should hit aaa bbb C#中国";
int results[5] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 3);
EXPECT_EQ(results[0], 28);
EXPECT_EQ(results[1], 27);
EXPECT_EQ(results[2], 18);
EXPECT_EQ(results[0], 139);
EXPECT_EQ(results[1], 138);
EXPECT_EQ(results[2], 129);
maat_state_free(&state);
}
TEST(maat_scan_ipv4, hit_ip_and_port) {
int table_id = maat_table_get_id(g_maat_instance, "IP_PLUS_CONFIG");
class MaatIPScan : public testing::Test
{
protected:
static void SetUpTestCase() {
table_id = maat_table_get_id(g_maat_instance, "IP_PLUS_CONFIG");
}
static void TearDownTestCase() {
}
static int table_id;
};
int MaatIPScan::table_id;
TEST_F(MaatIPScan, hit_ip_and_port) {
int table_id = MaatIPScan::table_id;
char ip_str[32] = "192.168.58.19";
uint32_t sip;
int ret = inet_pton(AF_INET, ip_str, &sip);
@@ -76,20 +106,20 @@ TEST(maat_scan_ipv4, hit_ip_and_port) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 7);
maat_state_free(&state);
state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
TEST(maat_scan_ipv4, hit_ip_and_port_range) {
#if 0
TEST_F(MaatStringScan, hit_ip_and_port_range) {
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "192.168.50.24";
uint32_t sip;
@@ -100,7 +130,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 4);
maat_state_free(&state);
@@ -109,7 +139,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
n_hit_result = 0;
state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 4);
maat_state_free(&state);
@@ -118,7 +148,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
n_hit_result = 0;
state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0);
}
@@ -135,7 +165,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 8);
maat_state_free(&state);
@@ -144,7 +174,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
EXPECT_EQ(ret, 1);
state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 8);
maat_state_free(&state);
@@ -153,7 +183,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
EXPECT_EQ(ret, 1);
state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
@@ -170,7 +200,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 50);
maat_state_free(&state);
@@ -179,7 +209,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) {
EXPECT_EQ(ret, 1);
state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
@@ -195,7 +225,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 2);
EXPECT_EQ(results[0], 63);
EXPECT_EQ(results[1], 67);
@@ -203,7 +233,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
@@ -219,14 +249,14 @@ TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 47);
maat_state_free(&state);
state = NULL;
ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
@@ -239,7 +269,7 @@ TEST(maat_scan_string, dynamic_config) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
@@ -255,7 +285,7 @@ TEST(maat_scan_string, dynamic_config) {
sleep(2);
state = NULL;
ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 9999);
maat_state_free(&state);
@@ -273,7 +303,7 @@ TEST(maat_scan_ip, dynamic_config) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
@@ -289,11 +319,13 @@ TEST(maat_scan_ip, dynamic_config) {
sleep(2);
state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 9998);
maat_state_free(&state);
}
#endif
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{
@@ -341,25 +373,22 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
return 0;
}
int main(int argc, char ** argv)
int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
struct log_handle *logger)
{
int ret=0;
::testing::InitGoogleTest(&argc, argv);
char json_iris_path[128] = {0};
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
struct log_handle *logger = log_handle_create("./tmp.log", 0);
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger);
EXPECT_NE(c, nullptr);
if (nullptr == c) {
return -1;
}
redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb");
EXPECT_NE(reply, nullptr);
if (nullptr == reply) {
return -1;
}
if ((access(json_iris_path, F_OK)) < 0) {
char tmp_iris_path[128] = {0};
@@ -367,11 +396,15 @@ int main(int argc, char ** argv)
size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1);
if (ret < 0) {
return -1;
}
ret = json2iris(json_buff, json_filename, c, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, logger);
EXPECT_NE(ret, -1);
if (ret < 0) {
return -1;
}
}
size_t total_line_cnt = 0;
@@ -381,7 +414,9 @@ int main(int argc, char ** argv)
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
long long server_time = maat_cmd_redis_server_time_s(c);
EXPECT_NE(server_time, -1);
if (server_time < 0) {
return -1;
}
absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger);
@@ -399,17 +434,47 @@ int main(int argc, char ** argv)
FREE(s_rule);
redisFree(c);
return 0;
}
int main(int argc, char ** argv)
{
int ret=0;
::testing::InitGoogleTest(&argc, argv);
struct log_handle *logger = log_handle_create("./maat_framework_gtest.log", 0);
if (NULL == logger) {
printf("create log handle failed.\n");
return -1;
}
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_GTEST, "write config to redis failed.");
log_handle_destroy(logger);
return -1;
}
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, logger);
g_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
ret=RUN_ALL_TESTS();
log_handle_destroy(g_maat_instance->logger);
if (NULL == g_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_GTEST, "create maat instance in MaatStringScan failed.");
log_handle_destroy(logger);
return -1;
}
ret=RUN_ALL_TESTS();
maat_free(g_maat_instance);
log_handle_destroy(logger);
return ret;
}
}

13
test/maat_input_mode_gtest.cpp
View File

@@ -1,4 +1,3 @@
#include "utils.h"
#include "maat/maat.h"
#include "maat_utils.h"
#include "maat_rule.h"
@@ -11,10 +10,6 @@
const char *table_info_path = "./table_info.conf";
const char *json_filename = "maat_json.json";
TEST(EQ_Test, Always_True) {
EXPECT_EQ(1, 1);
}
TEST(json_mode, maat_scan_string) {
char json_iris_path[128] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
@@ -38,7 +33,7 @@ TEST(json_mode, maat_scan_string) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_HIT);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191);
@@ -81,7 +76,7 @@ TEST(iris_mode, maat_scan_string) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_HIT);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191);
@@ -201,7 +196,7 @@ TEST(redis_mode, maat_scan_string) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_HIT);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191);
@@ -217,4 +212,4 @@ int main(int argc, char ** argv)
::testing::InitGoogleTest(&argc, argv);
ret=RUN_ALL_TESTS();
return ret;
}
}

19
test/rcu_hash_gtest.cpp
View File

@@ -1,5 +1,5 @@
#include "../src/inc_internal/rcu_hash.h"
#include "../include/utils.h"
#include "rcu_hash.h"
#include "maat_utils.h"
#include <gtest/gtest.h>
@@ -47,9 +47,6 @@ TEST(rcu_hash_add_one_node, single_thread) {
ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 1);
ret = rcu_hash_updating_flag(htable);
EXPECT_EQ(ret, 1);
rcu_hash_commit(htable);
/* find in hash after commit */
@@ -69,9 +66,6 @@ TEST(rcu_hash_add_one_node, single_thread) {
ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 0);
ret = rcu_hash_updating_flag(htable);
EXPECT_EQ(ret, 0);
rcu_hash_free(htable);
}
@@ -112,9 +106,6 @@ TEST(rcu_hash_add_multi_node, single_thread) {
ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 2);
ret = rcu_hash_updating_flag(htable);
EXPECT_EQ(ret, 1);
rcu_hash_commit(htable);
/* find in hash after commit */
@@ -141,9 +132,6 @@ TEST(rcu_hash_add_multi_node, single_thread) {
ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 0);
ret = rcu_hash_updating_flag(htable);
EXPECT_EQ(ret, 0);
rcu_hash_free(htable);
}
@@ -181,9 +169,6 @@ TEST(rcu_hash_del_one_node, single_thread) {
ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 0);
ret = rcu_hash_updating_flag(htable);
EXPECT_EQ(ret, 1);
rcu_hash_commit(htable);
/* find in hash after commit */

8
test/table_info.conf
View File

@@ -68,7 +68,7 @@
"valid_column":3,
"custom": {
"group_id":1,
"superior_group_id":2
"super_group_id":2
}
},
{
@@ -125,12 +125,6 @@
"sport_format":7,
"sport1":8,
"sport2":9,
"daddr_format":10,
"dip1":11,
"dip2":12,
"dport_format":13,
"dport1":14,
"dport2":15,
"proto":16,
"direction":17
}

5
tools/maat_redis_tool.cpp
View File

@@ -5,7 +5,6 @@
#include <sys/types.h>
#include <sys/stat.h>
#include "utils.h"
#include "maat_rule.h"
#include "maat_utils.h"
#include "maat_command.h"
@@ -70,7 +69,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char
int i = 0;
int ret = 0;
int line_count = 0;
int update_type = CM_UPDATE_TYPE_INC;
int update_type = MAAT_UPDATE_TYPE_INC;
long long version = 0;
const char *cur_table = NULL;
char foreign_files_dir[256] = {0};
@@ -95,7 +94,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char
return;
}
assert(update_type == CM_UPDATE_TYPE_FULL);
assert(update_type == MAAT_UPDATE_TYPE_FULL);
printf("MAAT Version: %lld, key number: %d\n", version, rule_num);
if (0 == rule_num) {
goto clean_up;