gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unfinished work

Browse Source
This commit is contained in:
liuwentan
2023-02-03 17:28:14 +08:00
parent cca7d882e1
commit 57f0a0581a
45 changed files with 2338 additions and 1522 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
include/maat/maat.h
View File

@@ -32,33 +32,22 @@ struct maat_hit_path {
int item_id; int item_id;
int sub_group_id; int sub_group_id;
int top_group_id; int top_group_id;
int virtual_table_id; // 0 is not a virtual table. int vtable_id; // 0 is not a virtual table.
int compile_id; int compile_id;
}; };
struct maat_hit_object { struct maat_hit_object {
int virtual_table_id; int vtable_id;
int group_id; int group_id;
}; };
enum maat_scan_opt enum maat_scan_status {
{ MAAT_SCAN_ERR = -1, //scan error
MAAT_SET_SCAN_DISTRICT = 1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default. MAAT_SCAN_OK, //scan but not hit(group or compile)
MAAT_SET_SCAN_LAST_ITEM, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination. MAAT_SCAN_HALF_HIT, //half hit: hit group, not hit compile
MAAT_SET_SCAN_COMPILE_TABLE_ID, //Caller can specify which compile table to use MAAT_SCAN_HIT //scan hit compile
MAAT_GET_SCAN_HIT_PATH, //VALUE is struct maat_hit_path*, an array of struct maat_hit_path, SIZE= sizeof(struct maat_hit_path)*N,
//Maat_get_scan_status returns actual got number.
MAAT_GET_SCAN_HIT_OBJECTS //VALUE is struct maat_hit_object*, an array of struct maat_hit_object, SIZE= sizeof(struct maat_hit_object)*N,
}; };
#define MAAT_OK 0 //scan but not hit(group or compile)
#define MAAT_ERR -1 //scan error
#define MAAT_HALF_HIT 1 //half hit: hit group, not hit compile
#define MAAT_HIT 2 //scan hit compile
#define MAAT_RULE_UPDATE_TYPE_FULL 1
#define MAAT_RULE_UPDATE_TYPE_INC 2
typedef void maat_start_callback_t(int update_type, void *u_param); typedef void maat_start_callback_t(int update_type, void *u_param);
typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para); typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para);
typedef void maat_finish_callback_t(void *u_para); typedef void maat_finish_callback_t(void *u_para);
@@ -125,8 +114,14 @@ struct maat_state;
* @param n_hit_result: the number of hit compile id * @param n_hit_result: the number of hit compile id
* @param state: scan mid status * @param state: scan mid status
* *
* @retval MAAT_ERR/MAAT_OK/MAAT_HALF_HIT * @retval MAAT_SCAN_ERR
* MAAT_SCAN_OK
* MAAT_SCAN_HALF_HIT
* MAAT_SCAN_HIT
*/ */
int maat_scan_flag(struct maat *instance, int table_id, int thread_id,
uint64_t flag, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
int maat_scan_integer(struct maat *instance, int table_id, int thread_id, int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, int *results, size_t n_result, unsigned int intval, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state); size_t *n_hit_result, struct maat_state **state);
@@ -152,15 +147,19 @@ int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* dat
void maat_scan_stream_close(struct maat_stream **stream); void maat_scan_stream_close(struct maat_stream **stream);
/* maat state API */ /* maat state API */
int maat_state_set_scan_district(struct maat *instance, struct maat_state **state, const char *district, size_t district_len); int maat_state_set_scan_district(struct maat *instance, struct maat_state **state,
const char *district, size_t district_len);
int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state); int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state);
int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, int compile_table_id); int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state,
int compile_table_id);
int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state, struct maat_hit_path *paths, size_t n_path); int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state,
struct maat_hit_path *paths, size_t n_path);
int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, struct maat_hit_object *objs, size_t n_obj); int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state,
struct maat_hit_object *objs, size_t n_obj);
void maat_state_free(struct maat_state **state); void maat_state_free(struct maat_state **state);

36
include/utils.h
View File

@@ -1,36 +0,0 @@
/*
**********************************************************************************************
* File: maat_utils.h
* Description: maat utils entry
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _UTILS_H_
#define _UTILS_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include <stdlib.h>
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
#define FREE(ptr) \
{ \
if (ptr) \
{ \
free(ptr); \
ptr = NULL; \
} \
}
#ifdef __cpluscplus
}
#endif
#endif

106
scanner/adapter_hs.cpp
View File

@@ -16,7 +16,6 @@
#include "adapter_hs.h" #include "adapter_hs.h"
#include "uthash/utarray.h" #include "uthash/utarray.h"
#include "uthash/uthash.h" #include "uthash/uthash.h"
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "bool_matcher.h" #include "bool_matcher.h"
@@ -43,7 +42,7 @@ struct adapter_hs_runtime {
/* adapter_hs instance */ /* adapter_hs instance */
struct adapter_hs { struct adapter_hs {
size_t nr_worker_threads; size_t n_worker_thread;
size_t n_expr; size_t n_expr;
size_t n_patterns; size_t n_patterns;
struct adapter_hs_runtime *hs_rt; struct adapter_hs_runtime *hs_rt;
@@ -59,25 +58,27 @@ struct adapter_hs_stream {
UT_array *pattern_id_set; UT_array *pattern_id_set;
}; };
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_threads, int max_pattern_type, static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt,
size_t n_worker_thread, int pattern_type,
struct log_handle *logger) struct log_handle *logger)
{ {
hs_database_t *database = NULL; hs_database_t *database = NULL;
hs_rt->scratchs = ALLOC(hs_scratch_t *, nr_worker_threads); hs_rt->scratchs = ALLOC(hs_scratch_t *, n_worker_thread);
if (max_pattern_type == PATTERN_TYPE_STR) { if (pattern_type == PATTERN_TYPE_STR) {
database = hs_rt->literal_db; database = hs_rt->literal_db;
} else { } else {
database = hs_rt->regex_db; database = hs_rt->regex_db;
} }
if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) { if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS, "ERROR: Unable to allocate scratch space. Exiting."); log_error(logger, MODULE_ADAPTER_HS,
"ERROR: Unable to allocate scratch space. Exiting.");
hs_free_database(database); hs_free_database(database);
return -1; return -1;
} }
for (size_t i = 1; i < nr_worker_threads; i++) { for (size_t i = 1; i < n_worker_thread; i++) {
hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]); hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS, "Unable to clone scratch prototype"); log_error(logger, MODULE_ADAPTER_HS, "Unable to clone scratch prototype");
@@ -119,7 +120,8 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
scan_mode, NULL, &hs_rt->literal_db, &compile_err); scan_mode, NULL, &hs_rt->literal_db, &compile_err);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
if (compile_err) { if (compile_err) {
log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message); log_error(logger, MODULE_ADAPTER_HS,
"%s compile error: %s", __func__, compile_err->message);
} }
hs_free_compile_error(compile_err); hs_free_compile_error(compile_err);
@@ -133,7 +135,8 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
scan_mode, NULL, &hs_rt->regex_db, &compile_err); scan_mode, NULL, &hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
if (compile_err) { if (compile_err) {
log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message); log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s",
__func__, compile_err->message);
} }
hs_free_compile_error(compile_err); hs_free_compile_error(compile_err);
goto error; goto error;
@@ -186,12 +189,14 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt
FREE(hs_cd); FREE(hs_cd);
} }
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array, struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t n_worker_thread,
and_expr_t *expr_array, size_t n_expr_array,
struct log_handle *logger) struct log_handle *logger)
{ {
if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) || if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) ||
0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) { 0 == n_worker_thread || NULL == expr_array || 0 == n_expr_array) {
log_error(logger, MODULE_ADAPTER_HS, "%s input parameters illegal!", __func__); log_error(logger, MODULE_ADAPTER_HS,
"%s input parameters illegal!", __func__);
return NULL; return NULL;
} }
@@ -202,7 +207,8 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
for (size_t i = 0; i < n_expr_array; i++) { for (size_t i = 0; i < n_expr_array; i++) {
if (expr_array[i].n_patterns > MAX_EXPR_PATTERN_NUM) { if (expr_array[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
log_error(logger, MODULE_ADAPTER_HS, log_error(logger, MODULE_ADAPTER_HS,
"the number of patterns in one expression should less than %d", MAX_EXPR_PATTERN_NUM); "the number of patterns in one expression should less than %d",
MAX_EXPR_PATTERN_NUM);
return NULL; return NULL;
} }
@@ -212,7 +218,8 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
} else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) { } else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) {
regex_pattern_num++; regex_pattern_num++;
} else { } else {
log_error(logger, MODULE_ADAPTER_HS, "unknown pattern type: %d", expr_array[i].patterns[j].type); log_error(logger, MODULE_ADAPTER_HS, "unknown pattern type: %d",
expr_array[i].patterns[j].type);
return NULL; return NULL;
} }
} }
@@ -283,7 +290,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
size_t mem_size = 0; size_t mem_size = 0;
struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1); struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1);
hs_instance->nr_worker_threads = nr_worker_threads; hs_instance->n_worker_thread = n_worker_thread;
hs_instance->n_patterns = pattern_id; hs_instance->n_patterns = pattern_id;
hs_instance->n_expr = n_expr_array; hs_instance->n_expr = n_expr_array;
hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1); hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
@@ -321,7 +328,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
max_patterns_type = PATTERN_TYPE_REG; max_patterns_type = PATTERN_TYPE_REG;
} }
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, nr_worker_threads, max_patterns_type, logger); ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, max_patterns_type, logger);
if (ret < 0) { if (ret < 0) {
goto error; goto error;
} }
@@ -349,7 +356,7 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance)
} }
if (hs_instance->hs_rt->scratchs != NULL) { if (hs_instance->hs_rt->scratchs != NULL) {
for (size_t i = 0; i < hs_instance->nr_worker_threads; i++) { for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
if (hs_instance->hs_rt->scratchs[i] != NULL) { if (hs_instance->hs_rt->scratchs[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->scratchs[i]); hs_free_scratch(hs_instance->hs_rt->scratchs[i]);
} }
@@ -399,10 +406,13 @@ int matched_event_cb(unsigned int id, unsigned long long from,
return 0; return 0;
} }
int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *data, size_t data_len, int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
int results[], size_t *n_results) const char *data, size_t data_len,
struct hs_scan_result *results,
size_t *n_results)
{ {
if (NULL == hs_instance || NULL == data || (0 == data_len) || NULL == results || NULL == n_results) { if (NULL == hs_instance || NULL == data || (0 == data_len) ||
NULL == results || NULL == n_results) {
return -1; return -1;
} }
@@ -416,7 +426,8 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d
int err_count = 0; int err_count = 0;
if (hs_rt->literal_db != NULL) { if (hs_rt->literal_db != NULL) {
err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set); err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch,
matched_event_cb, pattern_id_set);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
//log_error() //log_error()
err_count++; err_count++;
@@ -424,7 +435,8 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d
} }
if (hs_rt->regex_db != NULL) { if (hs_rt->regex_db != NULL) {
err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set); err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch,
matched_event_cb, pattern_id_set);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
//log_error() //log_error()
err_count++; err_count++;
@@ -442,18 +454,28 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d
items[i] = *(unsigned long long *)utarray_eltptr(pattern_id_set, i); items[i] = *(unsigned long long *)utarray_eltptr(pattern_id_set, i);
} }
size_t matched_index = 0; int ret = 0;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr); int matched_index = 0;
size_t bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_instance->n_expr);
struct bool_expr_match *bool_matcher_results = NULL;
bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr);
int bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, pattern_set_size,
bool_matcher_results, hs_instance->n_expr);
if (bool_matcher_ret < 0) {
ret = -1;
goto next;
}
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) { for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
results[matched_index] = bool_matcher_results[matched_index].expr_id; results[matched_index].item_id = bool_matcher_results[matched_index].expr_id;
results[matched_index].user_tag = bool_matcher_results[matched_index].user_tag;
} }
*n_results = bool_matcher_ret; *n_results = bool_matcher_ret;
next:
FREE(bool_matcher_results); FREE(bool_matcher_results);
utarray_free(pattern_id_set); utarray_free(pattern_id_set);
return 0; return ret;
} }
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id) struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id)
@@ -487,14 +509,16 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
return hs_stream; return hs_stream;
} }
int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len, int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream,
const char *data, size_t data_len,
int results[], size_t *n_results) int results[], size_t *n_results)
{ {
hs_error_t err; hs_error_t err;
int thread_id = hs_stream->thread_id; int thread_id = hs_stream->thread_id;
if (hs_stream->literal_stream != NULL) { if (hs_stream->literal_stream != NULL) {
err = hs_scan_stream(hs_stream->literal_stream, data, data_len, 0, hs_stream->hs_rt->scratchs[thread_id], err = hs_scan_stream(hs_stream->literal_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id],
matched_event_cb, hs_stream->pattern_id_set); matched_event_cb, hs_stream->pattern_id_set);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
//log_error() //log_error()
@@ -503,7 +527,8 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
} }
if (hs_stream->regex_stream != NULL) { if (hs_stream->regex_stream != NULL) {
err = hs_scan_stream(hs_stream->regex_stream, data, data_len, 0, hs_stream->hs_rt->scratchs[thread_id], err = hs_scan_stream(hs_stream->regex_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id],
matched_event_cb, hs_stream->pattern_id_set); matched_event_cb, hs_stream->pattern_id_set);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
//log_error() //log_error()
@@ -519,8 +544,10 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
} }
size_t matched_index = 0; size_t matched_index = 0;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr); struct bool_expr_match *bool_matcher_results = NULL;
size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_stream->n_expr); bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size,
bool_matcher_results, hs_stream->n_expr);
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) { for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
results[matched_index] = bool_matcher_results[matched_index].expr_id; results[matched_index] = bool_matcher_results[matched_index].expr_id;
} }
@@ -533,10 +560,19 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream) void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
{ {
if (NULL == hs_stream) {
return;
}
int thread_id = hs_stream->thread_id; int thread_id = hs_stream->thread_id;
hs_close_stream(hs_stream->literal_stream, hs_stream->hs_rt->scratchs[thread_id], NULL, NULL); if (hs_stream->hs_rt != NULL) {
hs_close_stream(hs_stream->regex_stream, hs_stream->hs_rt->scratchs[thread_id], NULL, NULL); hs_close_stream(hs_stream->literal_stream,
hs_stream->hs_rt->scratchs[thread_id], NULL, NULL);
hs_close_stream(hs_stream->regex_stream,
hs_stream->hs_rt->scratchs[thread_id], NULL, NULL);
}
utarray_free(hs_stream->pattern_id_set); utarray_free(hs_stream->pattern_id_set);
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */ /* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */

11
scanner/adapter_hs.h
View File

@@ -38,6 +38,11 @@ enum pattern_type {
PATTERN_TYPE_REG, PATTERN_TYPE_REG,
}; };
struct hs_scan_result {
int item_id;
void *user_tag;
};
typedef struct { typedef struct {
/* pattern type */ /* pattern type */
int type; int type;
@@ -53,6 +58,7 @@ typedef struct {
uint32_t expr_id; uint32_t expr_id;
size_t n_patterns; size_t n_patterns;
scan_pattern_t patterns[MAX_EXPR_PATTERN_NUM]; scan_pattern_t patterns[MAX_EXPR_PATTERN_NUM];
void *user_tag;
} and_expr_t; } and_expr_t;
/** /**
@@ -65,7 +71,8 @@ typedef struct {
* *
* @retval the pointer to adapter_hs instance * @retval the pointer to adapter_hs instance
*/ */
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array, struct log_handle *logger); struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array,
size_t n_expr_array, struct log_handle *logger);
/** /**
* @brief scan input data to match logic AND expression, return all matched expr_id * @brief scan input data to match logic AND expression, return all matched expr_id
@@ -78,7 +85,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
* @param n_results: number of elements in array of expr_id * @param n_results: number of elements in array of expr_id
*/ */
int adapter_hs_scan(struct adapter_hs *instance, int thread_id, const char *data, size_t data_len, int adapter_hs_scan(struct adapter_hs *instance, int thread_id, const char *data, size_t data_len,
int results[], size_t *n_results); struct hs_scan_result *results, size_t *n_results);
/** /**
* @brief destroy adapter_hs instance * @brief destroy adapter_hs instance

6
scanner/adapter_hs_gtest.cpp
View File

@@ -1,6 +1,5 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "utils.h"
#include "adapter_hs.h" #include "adapter_hs.h"
int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr) int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr)
@@ -33,7 +32,7 @@ int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr)
char *sub_pattern_token = strtok_r(pattern_token, ":", &save_pattern_ptr); char *sub_pattern_token = strtok_r(pattern_token, ":", &save_pattern_ptr);
expr[i].patterns[j].type = atoi(sub_pattern_token); expr[i].patterns[j].type = atoi(sub_pattern_token);
size_t str_len = strlen(save_pattern_ptr); size_t str_len = strlen(save_pattern_ptr);
expr[i].patterns[j].pat = ALLOC(char, str_len); expr[i].patterns[j].pat = (char *)calloc(sizeof(char), str_len);
memcpy(expr[i].patterns[j].pat, save_pattern_ptr, str_len); memcpy(expr[i].patterns[j].pat, save_pattern_ptr, str_len);
expr[i].patterns[j].pat_len = str_len; expr[i].patterns[j].pat_len = str_len;
j++; j++;
@@ -50,7 +49,8 @@ void expr_array_free(and_expr_t expr_array[], size_t n_expr_array)
{ {
for (size_t i = 0; i < n_expr_array; i++) { for (size_t i = 0; i < n_expr_array; i++) {
for (size_t j = 0; j < expr_array[i].n_patterns; j++) { for (size_t j = 0; j < expr_array[i].n_patterns; j++) {
FREE(expr_array[i].patterns[j].pat); free(expr_array[i].patterns[j].pat);
expr_array[i].patterns[j].pat = NULL;
} }
} }
} }

5
src/inc_internal/json2iris.h
View File

@@ -18,8 +18,9 @@ extern "C"
#include "hiredis/hiredis.h" #include "hiredis/hiredis.h"
int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx, int json2iris(const char *json_buff, const char *json_filename,
char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo, redisContext *redis_write_ctx, char *iris_dir_buf,
int buf_len, char *encrypt_key, char *encrypt_algo,
struct log_handle *logger); struct log_handle *logger);
#ifdef __cpluscplus #ifdef __cpluscplus

43
src/inc_internal/maat_compile.h
View File

@@ -33,14 +33,17 @@ struct compile_ex_data_schema {
}; };
/* compile schema API */ /* compile schema API */
void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void compile_schema_free(void *compile_schema); void compile_schema_free(void *compile_schema);
void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void group2compile_schema_free(void *g2c_schema); void group2compile_schema_free(void *g2c_schema);
int group2compile_associated_compile_table_id(void *g2c_schema); int group2compile_associated_compile_table_id(void *g2c_schema);
int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id, int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema,
int table_id,
maat_rule_ex_new_func_t *new_func, maat_rule_ex_new_func_t *new_func,
maat_rule_ex_free_func_t *free_func, maat_rule_ex_free_func_t *free_func,
maat_rule_ex_dup_func_t *dup_func, maat_rule_ex_dup_func_t *dup_func,
@@ -52,37 +55,43 @@ compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, siz
size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema); size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema);
/* compile runtime API */ /* compile runtime API */
void *compile_runtime_new(void *compile_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *compile_runtime_new(void *compile_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger); struct log_handle *logger);
void compile_runtime_free(void *compile_runtime); void compile_runtime_free(void *compile_runtime);
int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, int compile_runtime_update(void *compile_runtime, void *compile_schema,
int valid_column); const char *line, int valid_column);
int compile_runtime_commit(void *compile_runtime); int compile_runtime_commit(void *compile_runtime);
int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids, int compile_runtime_match(struct compile_runtime *compile_rt, int *compile_ids,
int vt_id, int *compile_ids, size_t compile_ids_size, struct maat_state *state); size_t compile_ids_size, struct maat_state *state);
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct group2group_runtime *g2g_rt, size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
struct group2group_runtime *g2g_rt,
struct maat_compile_state *compile_state, struct maat_compile_state *compile_state,
struct maat_hit_path *hit_paths, size_t hit_path_siz); struct maat_hit_path *hit_paths,
size_t n_hit_path);
/* group2compile runtime API */ /* group2compile runtime API */
void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *group2compile_runtime_new(void *g2c_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger); struct log_handle *logger);
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime); void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime);
void group2compile_runtime_free(void *g2c_runtime); void group2compile_runtime_free(void *g2c_runtime);
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line, int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
int valid_column); const char *line, int valid_column);
/* maat compile state API */ /* maat compile state API */
struct maat_compile_state; struct maat_compile_state;
struct maat_compile_state *maat_compile_state_new(int thread_id); struct maat_compile_state *maat_compile_state_new(int thread_id);
void maat_compile_state_free(struct maat_compile_state *compile_state); void maat_compile_state_free(struct maat_compile_state *compile_state);
void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id,
int Nth_scan, int Nth_item_result); int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, void *compile_runtime, int *hit_item_ids, size_t hit_item_cnt,
int group_id, int virtual_table_id); int *group_ids, size_t group_ids_size,
size_t *n_hit_group_id, struct maat_state *state);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);
#ifdef __cpluscplus #ifdef __cpluscplus

7
src/inc_internal/maat_config_monitor.h
View File

@@ -18,17 +18,14 @@ extern "C"
#include <stdint.h> #include <stdint.h>
#define CM_UPDATE_TYPE_NONE 0
#define CM_UPDATE_TYPE_FULL 1
#define CM_UPDATE_TYPE_INC 2
void config_monitor_traverse(long long version, const char *idx_dir, void config_monitor_traverse(long long version, const char *idx_dir,
void (*start_fn)(long long, int, void *), void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *), int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *), void *u_param, void (*finish_fn)(void *), void *u_param,
struct log_handle *logger); struct log_handle *logger);
int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz); int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
char *err_str, size_t err_str_sz);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

28
src/inc_internal/maat_ex_data.h
View File

@@ -33,11 +33,10 @@ struct ex_container_ctx {
struct ex_data_runtime; struct ex_data_runtime;
/* ex_data_runtime API */ /* ex_data_runtime API */
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn, struct ex_data_runtime *
struct log_handle *logger); ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn,
struct log_handle *logger);
void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt); void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt);
void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt); void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt);
/* ex_data_runtime cache row API */ /* ex_data_runtime cache row API */
@@ -56,19 +55,22 @@ struct ex_data_schema *ex_data_schema_new(maat_plugin_ex_new_func_t *new_func,
long argl, void *argp); long argl, void *argp);
void ex_data_schema_free(struct ex_data_schema *ex_schema); void ex_data_schema_free(struct ex_data_schema *ex_schema);
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema); void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt,
struct ex_data_schema *schema);
/* set user_ctx API */ /* set user_ctx API */
void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, struct ex_container_ctx *container_ctx); void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt,
struct ex_container_ctx *container_ctx);
struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt); struct ex_container_ctx *
ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt);
struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data); struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data);
void ex_data_container_free(void *ctx, void *data); void ex_data_container_free(void *ctx, void *data);
/* ex_data_runtime ex data API */ /* ex_data_runtime ex data API */
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row,
const char *row, const char *key, size_t key_len); const char *key, size_t key_len);
int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len, const char *key, size_t key_len,
@@ -80,14 +82,14 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container ***ex_container); struct ex_data_container ***ex_container);
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt); size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt);
int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt);
#ifdef __cpluscplus #ifdef __cpluscplus
} }
#endif #endif

24
src/inc_internal/maat_expr.h
View File

@@ -21,16 +21,18 @@ extern "C"
struct expr_runtime; struct expr_runtime;
void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void expr_schema_free(void *expr_schema); void expr_schema_free(void *expr_schema);
/* expr runtime API */ /* expr runtime API */
void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *expr_runtime_new(void *expr_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger); struct log_handle *logger);
void expr_runtime_free(void *expr_runtime); void expr_runtime_free(void *expr_runtime);
int expr_runtime_updating_flag(void *expr_runtime); int expr_runtime_update(void *expr_runtime, void *expr_schema,
int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column); const char *line, int valid_column);
int expr_runtime_commit(void *expr_runtime); int expr_runtime_commit(void *expr_runtime);
/* expr runtime scan API */ /* expr runtime scan API */
@@ -39,15 +41,19 @@ int expr_runtime_commit(void *expr_runtime);
* *
* @retval the num of hit group_id * @retval the num of hit group_id
*/ */
int expr_runtime_scan_string(struct expr_runtime *expr_runtime, int thread_id, int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len, int group_ids[], const char *data, size_t data_len,
size_t max_hit_num, int virtual_table_id, int *group_ids, size_t group_ids_size,
struct maat_state *state); int vtable_id, struct maat_state *state);
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, int results[], size_t *n_result); int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data,
size_t data_len, int result[], size_t *n_result);
void expr_runtime_stream_close(struct expr_runtime *expr_rt); void expr_runtime_stream_close(struct expr_runtime *expr_rt);
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id);
long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread);
#ifdef __cpluscplus #ifdef __cpluscplus
} }
#endif #endif

3
src/inc_internal/maat_garbage_collection.h
View File

@@ -24,7 +24,8 @@ struct maat_garbage_bin *maat_garbage_bin_new(int default_timeout);
void maat_garbage_bin_free(struct maat_garbage_bin *bin); void maat_garbage_bin_free(struct maat_garbage_bin *bin);
void maat_garbage_bagging(struct maat_garbage_bin *bin, void *garbage, void (* func)(void *)); void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage,
void (* func)(void *));
void maat_garbage_collect_routine(struct maat_garbage_bin *bin); void maat_garbage_collect_routine(struct maat_garbage_bin *bin);

53
src/inc_internal/maat_group.h
View File

@@ -21,63 +21,34 @@ extern "C"
#include "igraph/igraph.h" #include "igraph/igraph.h"
#include "maat_kv.h" #include "maat_kv.h"
struct maat_group { struct maat_group;
igraph_integer_t vertex_id;
int group_id;
int ref_by_compile_cnt;
int ref_by_superior_group_cnt;
int ref_by_subordinate_group_cnt;
size_t top_group_cnt;
int *top_group_ids;
UT_hash_handle hh_group_id;
UT_hash_handle hh_vertex_id;
};
/* maat group topology API */
struct maat_group_topology;
/**
* @retval if not found, return NULL
*/
//struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id);
//struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id);
//void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group);
//int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
//int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
/* build top groups */
//int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo);
/* group2group schema API */ /* group2group schema API */
void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void group2group_schema_free(void *g2g_schema); void group2group_schema_free(void *g2g_schema);
/* group2group runtime API */ /* group2group runtime API */
void *group2group_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *group2group_runtime_new(void *g2g_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger); struct log_handle *logger);
void group2group_runtime_free(void *g2g_runtime); void group2group_runtime_free(void *g2g_runtime);
void maat_group_ref_inc(struct maat_group *group);
void maat_group_ref_dec(struct maat_group *group);
struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id); struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id);
void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group); void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group);
struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id); struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id);
int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int superior_group_id);
int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, int superior_group_id);
int group2group_runtime_build_top_groups(void *g2g_runtime); int group2group_runtime_build_top_groups(void *g2g_runtime);
int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids,
size_t n_group_ids, int *top_group_ids);
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column); const char *line, int valid_column);
int group2group_runtime_commit(void *g2g_runtime); int group2group_runtime_commit(void *g2g_runtime);
int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t n_group_ids,
int *top_group_ids);
#ifdef __cpluscplus #ifdef __cpluscplus
} }
#endif #endif

34
src/inc_internal/maat_ip.h
View File

@@ -16,27 +16,31 @@ extern "C"
{ {
#endif #endif
struct ip_plus_runtime; struct ip_runtime;
void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void ip_plus_schema_free(void *ip_plus_schema); const char *table_name, struct log_handle *logger);
void ip_schema_free(void *ip_schema);
/* ip plus runtime API */ /* ip runtime API */
void *ip_plus_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *ip_runtime_new(void *ip_schema, int max_thread_num,
struct log_handle *logger); struct maat_garbage_bin *garbage_bin,
void ip_plus_runtime_free(void *ip_plus_runtime); struct log_handle *logger);
void ip_runtime_free(void *ip_runtime);
int ip_plus_runtime_updating_flag(void *ip_plus_runtime); int ip_runtime_update(void *ip_runtime, void *ip_schema,
int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line, const char *line, int valid_column);
int valid_column); int ip_runtime_commit(void *ip_runtime);
int ip_plus_runtime_commit(void *ip_plus_runtime);
struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt); struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt);
/* ip runtime scan API */ /* ip runtime scan API */
int ip_plus_runtime_scan_ipv4(struct ip_plus_runtime *ip_plus_rt, int thread_id, uint32_t ip_addr, int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, int ip_type,
int *group_id_array, size_t n_group_id_array, int virtual_table_id, uint8_t *ip_addr, int *group_ids, size_t group_id_size,
struct maat_state *state); int vtable_id, struct maat_state *state);
void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id);
long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

11
src/inc_internal/maat_ip_plugin.h
View File

@@ -23,7 +23,8 @@ extern "C"
struct ip_plugin_runtime; struct ip_plugin_runtime;
/* ip plugin schema API */ /* ip plugin schema API */
void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void ip_plugin_schema_free(void *ip_plugin_schema); void ip_plugin_schema_free(void *ip_plugin_schema);
/* ip plugin table ex data API */ /* ip plugin table ex data API */
@@ -37,13 +38,13 @@ int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
struct log_handle *logger); struct log_handle *logger);
/* ip plugin runtime API */ /* ip plugin runtime API */
void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger); struct log_handle *logger);
void ip_plugin_runtime_free(void *ip_plugin_runtime); void ip_plugin_runtime_free(void *ip_plugin_runtime);
int ip_plugin_runtime_updating_flag(void *ip_plugin_runtime); int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line, const char *line, int valid_column);
int valid_column);
int ip_plugin_runtime_commit(void *ip_plugin_runtime); int ip_plugin_runtime_commit(void *ip_plugin_runtime);
struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime); struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime);

24
src/inc_internal/maat_plugin.h
View File

@@ -21,7 +21,8 @@ extern "C"
#define MAX_FOREIGN_CLMN_NUM 8 #define MAX_FOREIGN_CLMN_NUM 8
/* plugin schema API */ /* plugin schema API */
void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void plugin_schema_free(void *plugin_schema); void plugin_schema_free(void *plugin_schema);
struct plugin_schema; struct plugin_schema;
@@ -38,23 +39,22 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema);
int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns); int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns);
/* plugin table ex data API */ /* plugin table ex data API */
int plugin_table_set_ex_data_schema(void *custom_schema, int plugin_table_set_ex_data_schema(void *plugin_schema,
maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func, maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp, long argl, void *argp,
struct log_handle *logger); struct log_handle *logger);
struct ex_data_schema *plugin_table_get_ex_data_schema(void *custom_schema); struct ex_data_schema *plugin_table_get_ex_data_schema(void *custom_schema);
/* plugin runtime API */ /* plugin runtime API */
void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *plugin_runtime_new(void *plugin_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger); struct log_handle *logger);
void plugin_runtime_free(void *plugin_runtime); void plugin_runtime_free(void *plugin_runtime);
int plugin_runtime_updating_flag(void *plugin_runtime); int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
const char *line, int valid_column);
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line,
int valid_column);
int plugin_runtime_commit(void *plugin_runtime); int plugin_runtime_commit(void *plugin_runtime);
struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime); struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime);

2
src/inc_internal/maat_redis_monitor.h
View File

@@ -20,7 +20,7 @@ extern "C"
#include <stdint.h> #include <stdint.h>
void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx, void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
void (*start_fn)(long long, int, void *), void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *), int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *), void *u_param); void (*finish_fn)(void *), void *u_param);

65
src/inc_internal/maat_rule.h
View File

@@ -34,6 +34,22 @@ extern "C"
#define MAX_TABLE_NUM 256 #define MAX_TABLE_NUM 256
#define MAAT_UPDATE_TYPE_NONE 0
#define MAAT_UPDATE_TYPE_FULL 1
#define MAAT_UPDATE_TYPE_INC 2
enum last_scan_flag {
LAST_SCAN_UNSET,
LAST_SCAN_SET,
LAST_SCAN_FINISHED
};
enum tag_match {
TAG_MATCH_ERR = -1,
TAG_MATCH_UNMATCHED,
TAG_MATCH_MATCHED
};
struct maat_rule_head { struct maat_rule_head {
int config_id; int config_id;
int service_id; int service_id;
@@ -89,7 +105,7 @@ struct compile_rule {
struct group2group_rule { struct group2group_rule {
int group_id; int group_id;
int superior_group_id; int super_group_id;
}; };
struct maat_runtime { struct maat_runtime {
@@ -223,9 +239,11 @@ struct maat {
/* statistics */ /* statistics */
long long line_cmd_acc_num; long long line_cmd_acc_num;
long long *outer_mid_cnt; long long *outer_state_cnt;
long long *compile_mid_cnt; long long *compile_state_cnt;
long long *thread_call_cnt; long long *thread_call_cnt;
long long *hit_cnt;
long long *not_grp_hit_cnt;
long long scan_err_cnt; long long scan_err_cnt;
}; };
@@ -238,7 +256,7 @@ struct maat_state {
unsigned char is_last_scan; unsigned char is_last_scan;
int district_id; //-1: Any District; -2: Unkonwn District; int district_id; //-1: Any District; -2: Unkonwn District;
int scan_cnt; int scan_cnt;
struct maat_compile_state *compile_mid; struct maat_compile_state *compile_state;
}; };
size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger); size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger);
@@ -264,7 +282,8 @@ void *rule_monitor_loop(void *arg);
void maat_read_full_config(struct maat *maat_instance); void maat_read_full_config(struct maat *maat_instance);
/* maat command API for internal */ /* maat command API for internal */
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db, struct log_handle *logger); redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger);
redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...); redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...);
@@ -274,35 +293,39 @@ long long maat_cmd_redis_server_time_s(redisContext *c);
long long maat_cmd_read_redis_integer(const redisReply *reply); long long maat_cmd_read_redis_integer(const redisReply *reply);
int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq); int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
int valid_column_seq);
const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len); const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len);
int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
long long server_time, struct log_handle *logger); size_t serial_rule_num, long long server_time,
struct log_handle *logger);
void maat_cmd_clear_rule_cache(struct serial_rule *s_rule); void maat_cmd_clear_rule_cache(struct serial_rule *s_rule);
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version,
long long *new_version, struct table_manager* tbl_mgr, long long desired_version, long long *new_version,
struct serial_rule **list, int *update_type, int cumulative_off, struct table_manager *tbl_mgr, struct serial_rule **list,
struct log_handle *logger); int *update_type, int cumulative_off, struct log_handle *logger);
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process, int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list,
struct log_handle *logger); int rule_num, int print_process, struct log_handle *logger);
int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list,
const char *dir, struct log_handle *logger); int rule_num, const char* dir, struct log_handle *logger);
void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn, void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
struct log_handle *logger); int rule_num, int print_fn, struct log_handle *logger);
void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule); void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule);
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id, void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
const char *table_name, const char *line, long long timeout); unsigned long rule_id, const char *table_name,
const char *line, long long timeout);
void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, const char *srv_def, int srv_def_len); void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head,
const char *srv_def, int srv_def_len);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

16
src/inc_internal/maat_table.h
View File

@@ -43,8 +43,9 @@ enum table_type {
struct table_manager; struct table_manager;
struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags, struct table_manager *
struct log_handle *logger); table_manager_create(const char *table_info_path, const char *accept_tags,
struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num, int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num,
struct maat_garbage_bin *garbage_bin); struct maat_garbage_bin *garbage_bin);
void table_manager_runtime_destroy(struct table_manager *tbl_mgr); void table_manager_runtime_destroy(struct table_manager *tbl_mgr);
@@ -53,14 +54,21 @@ void table_manager_destroy(struct table_manager *tbl_mgr);
size_t table_manager_table_count(struct table_manager *tbl_mgr); size_t table_manager_table_count(struct table_manager *tbl_mgr);
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name); int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name);
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id); enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id);
int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr); int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr);
int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr);
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id); int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);
size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr);
int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags); int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags);
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id); int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str,
size_t district_str_len, int *district_id);
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id); void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);
int table_manager_runtime_updating_flag(struct table_manager *tbl_mgr, int table_id);
int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line); int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line);
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id); void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id);

21
src/inc_internal/maat_utils.h
View File

@@ -31,6 +31,27 @@ extern "C"
#define MIN(a, b) (((a) < (b)) ? (a) : (b)) #define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif #endif
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
#define FREE(ptr) \
{ \
if (ptr) \
{ \
free(ptr); \
ptr = NULL; \
} \
}
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
#endif
#define MAX_SCANNER_HIT_COMPILE_NUM 4096 #define MAX_SCANNER_HIT_COMPILE_NUM 4096
#define MAX_SCANNER_HIT_GROUP_NUM 4096 #define MAX_SCANNER_HIT_GROUP_NUM 4096
#define MAX_SCANNER_HIT_ITEM_NUM 4096 #define MAX_SCANNER_HIT_ITEM_NUM 4096

3
src/inc_internal/maat_virtual.h
View File

@@ -29,7 +29,8 @@ enum scan_type {
SCAN_TYPE_MAX SCAN_TYPE_MAX
}; };
void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void virtual_schema_free(void *virtual_schema); void virtual_schema_free(void *virtual_schema);

7
src/inc_internal/rcu_hash.h
View File

@@ -62,13 +62,6 @@ size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable);
size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array); size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array);
/**
* @brief check if rcu hash table is updating
*
* @retval 1 means htable is updating, otherwise 0
*/
int rcu_hash_updating_flag(struct rcu_hash_table *htable);
#ifdef __cpluscplus #ifdef __cpluscplus
} }
#endif #endif

221
src/json2iris.cpp
View File

@@ -15,7 +15,6 @@
#include <errno.h> #include <errno.h>
#include <sys/stat.h> #include <sys/stat.h>
#include "utils.h"
#include "json2iris.h" #include "json2iris.h"
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
#include "maat_kv.h" #include "maat_kv.h"
@@ -23,7 +22,7 @@
#include "maat_rule.h" #include "maat_rule.h"
#include "uthash/uthash.h" #include "uthash/uthash.h"
#define MODULE_JSON2IRIS module_name_str("maat.json2iris") #define MODULE_JSON2IRIS module_name_str("maat.json2iris")
#define MAX_COLUMN_NUM 32 #define MAX_COLUMN_NUM 32
@@ -89,7 +88,9 @@ struct translate_command {
int default_int; int default_int;
}; };
struct iris_table *query_table_info(struct iris_description *p_iris, const char *table_name, enum table_type table_type) struct iris_table *query_table_info(struct iris_description *p_iris,
const char *table_name,
enum table_type table_type)
{ {
struct iris_table *table_info = NULL; struct iris_table *table_info = NULL;
HASH_FIND(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info); HASH_FIND(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info);
@@ -97,8 +98,10 @@ struct iris_table *query_table_info(struct iris_description *p_iris, const char
table_info = ALLOC(struct iris_table, 1); table_info = ALLOC(struct iris_table, 1);
table_info->line_count = 0; table_info->line_count = 0;
table_info->table_type = table_type; table_info->table_type = table_type;
memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name) - 1, strlen(table_name))); memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name) - 1,
snprintf(table_info->table_path, sizeof(table_info->table_path), "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name); strlen(table_name)));
snprintf(table_info->table_path, sizeof(table_info->table_path),
"%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name);
HASH_ADD_KEYPTR(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info); HASH_ADD_KEYPTR(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info);
} }
@@ -114,14 +117,18 @@ void free_iris_table_info(void *p)
FREE(table); FREE(table);
} }
int set_iris_descriptor(const char *json_file, cJSON *json, const char *encrypt_key, const char *encrypt_algo, int set_iris_descriptor(const char *json_file, cJSON *json,
const char *compile_tn, const char *group2compile_tn, const char* group2group_tn, const char *encrypt_key, const char *encrypt_algo,
redisContext *redis_write_ctx, struct iris_description *iris_cfg) const char *compile_tn, const char *group2compile_tn,
const char* group2group_tn, redisContext *redis_write_ctx,
struct iris_description *iris_cfg)
{ {
memset(iris_cfg, 0, sizeof(struct iris_description)); memset(iris_cfg, 0, sizeof(struct iris_description));
snprintf(iris_cfg->tmp_iris_dir, sizeof(iris_cfg->tmp_iris_dir), "%s_iris_tmp", json_file); snprintf(iris_cfg->tmp_iris_dir, sizeof(iris_cfg->tmp_iris_dir), "%s_iris_tmp", json_file);
snprintf(iris_cfg->tmp_iris_index_dir, sizeof(iris_cfg->tmp_iris_index_dir), "%s_iris_tmp/index", json_file); snprintf(iris_cfg->tmp_iris_index_dir, sizeof(iris_cfg->tmp_iris_index_dir),
snprintf(iris_cfg->index_path, sizeof(iris_cfg->index_path), "%s/full_config_index.%010d", iris_cfg->tmp_iris_index_dir, json_version); "%s_iris_tmp/index", json_file);
snprintf(iris_cfg->index_path, sizeof(iris_cfg->index_path), "%s/full_config_index.%010d",
iris_cfg->tmp_iris_index_dir, json_version);
iris_cfg->redis_write_ctx = redis_write_ctx; iris_cfg->redis_write_ctx = redis_write_ctx;
iris_cfg->str2int_map = maat_kv_store_new(); iris_cfg->str2int_map = maat_kv_store_new();
@@ -219,13 +226,15 @@ int create_tmp_dir(struct iris_description *p)
return 0; return 0;
} }
int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_description *p_iris, int write_plugin_line(cJSON *plug_table_json, int sequence,
struct iris_description *p_iris,
struct log_handle *logger) struct log_handle *logger)
{ {
cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name"); cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name");
if (NULL == item || item->type != cJSON_String) { if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_JSON2IRIS, log_error(logger, MODULE_JSON2IRIS,
"The %d plugin_table's table_name not defined or format error", sequence); "The %d plugin_table's table_name not defined or format error",
sequence);
return -1; return -1;
} }
const char *table_name = item->valuestring; const char *table_name = item->valuestring;
@@ -233,7 +242,8 @@ int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_descript
cJSON *table_content = cJSON_GetObjectItem(plug_table_json, "table_content"); cJSON *table_content = cJSON_GetObjectItem(plug_table_json, "table_content");
if (NULL == table_content || table_content->type != cJSON_Array) { if (NULL == table_content || table_content->type != cJSON_Array) {
log_error(logger, MODULE_JSON2IRIS, log_error(logger, MODULE_JSON2IRIS,
"%d plugin_table's table_content not defined or format error", sequence); "%d plugin_table's table_content not defined or format error",
sequence);
return -1; return -1;
} }
int line_cnt = cJSON_GetArraySize(table_content); int line_cnt = cJSON_GetArraySize(table_content);
@@ -244,21 +254,25 @@ int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_descript
for (int i = 0; i < line_cnt; i++) { for (int i = 0; i < line_cnt; i++) {
each_line = cJSON_GetArrayItem(table_content, i); each_line = cJSON_GetArrayItem(table_content, i);
if (NULL == each_line || each_line->type != cJSON_String) { if (NULL == each_line || each_line->type != cJSON_String) {
log_error(logger, MODULE_JSON2IRIS, "plugin_table %s's line %d format error", log_error(logger, MODULE_JSON2IRIS,
table_info->table_name, i + 1); "plugin_table %s's line %d format error",
table_info->table_name, i + 1);
continue; continue;
} }
line_content = each_line->valuestring; line_content = each_line->valuestring;
table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), line_content, strlen(line_content)); table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos,
table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), "\n", 1); &(table_info->buff_sz), line_content, strlen(line_content));
table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos,
&(table_info->buff_sz), "\n", 1);
table_info->line_count++; table_info->line_count++;
} }
return 0; return 0;
} }
static struct group_info *group_info_read(struct group_info *group_name_map, const char *group_name) static struct group_info *group_info_read(struct group_info *group_name_map,
const char *group_name)
{ {
struct group_info *node = NULL; struct group_info *node = NULL;
HASH_FIND(hh, group_name_map, group_name, strlen(group_name), node); HASH_FIND(hh, group_name_map, group_name, strlen(group_name), node);
@@ -274,7 +288,8 @@ static int get_group_seq(struct iris_description *iris_cfg)
if (NULL == iris_cfg->redis_write_ctx) { if (NULL == iris_cfg->redis_write_ctx) {
sequence = iris_cfg->group_cnt; sequence = iris_cfg->group_cnt;
} else { } else {
data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx, "INCRBY %s 1", mr_group_id_var); data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx,
"INCRBY %s 1", mr_group_id_var);
sequence = (int)data_reply->integer - 1; sequence = (int)data_reply->integer - 1;
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply = NULL; data_reply = NULL;
@@ -284,7 +299,8 @@ static int get_group_seq(struct iris_description *iris_cfg)
return sequence; return sequence;
} }
static struct group_info *group_info_add_unsafe(struct iris_description *p_iris, const char *group_name) static struct group_info *group_info_add_unsafe(struct iris_description *p_iris,
const char *group_name)
{ {
static struct group_info untitled_group; static struct group_info untitled_group;
struct group_info *group_info = NULL; struct group_info *group_info = NULL;
@@ -308,7 +324,8 @@ static int get_region_seq(struct iris_description *iris_cfg)
if (NULL == iris_cfg->redis_write_ctx) { if (NULL == iris_cfg->redis_write_ctx) {
sequence = iris_cfg->region_cnt; sequence = iris_cfg->region_cnt;
} else { } else {
redisReply *data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx, "INCRBY %s 1", mr_region_id_var); redisReply *data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx,
"INCRBY %s 1", mr_region_id_var);
sequence = (int)data_reply->integer - 1; sequence = (int)data_reply->integer - 1;
freeReplyObject(data_reply); freeReplyObject(data_reply);
} }
@@ -317,7 +334,8 @@ static int get_region_seq(struct iris_description *iris_cfg)
return sequence; return sequence;
} }
int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct translate_command *cmd, int cmd_cnt, int direct_write_rule(cJSON *json, struct maat_kv_store *str2int,
struct translate_command *cmd, int cmd_cnt,
struct iris_table *table, struct log_handle *logger) struct iris_table *table, struct log_handle *logger)
{ {
int i = 0; int i = 0;
@@ -335,7 +353,8 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla
} }
if (NULL == item || item->type != cmd[i].json_type) { if (NULL == item || item->type != cmd[i].json_type) {
log_error(logger, MODULE_JSON2IRIS, "%s not defined or wrong format", cmd[i].json_string); log_error(logger, MODULE_JSON2IRIS,
"%s not defined or wrong format", cmd[i].json_string);
ret = -1; ret = -1;
goto error_out; goto error_out;
} }
@@ -345,7 +364,8 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla
char *p = item->valuestring; char *p = item->valuestring;
ret = maat_kv_read(str2int, p, &int_value); ret = maat_kv_read(str2int, p, &int_value);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_JSON2IRIS, "%s's value %s is not valid format", cmd[i].json_string, p); log_error(logger, MODULE_JSON2IRIS,
"%s's value %s is not valid format", cmd[i].json_string, p);
FREE(p); FREE(p);
ret = -1; ret = -1;
goto error_out; goto error_out;
@@ -369,7 +389,8 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla
} }
for (i = 0; i < cmd_cnt; i++) { for (i = 0; i < cmd_cnt; i++) {
table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, cmd[i].json_value, strlen(cmd[i].json_value)); table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz,
cmd[i].json_value, strlen(cmd[i].json_value));
table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1); table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1);
} }
table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1); table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1);
@@ -384,8 +405,8 @@ error_out:
return ret; return ret;
} }
int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, int write_expr_line(cJSON *region_json, struct iris_description *p_iris,
struct log_handle *logger) struct iris_table *table, struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -428,11 +449,12 @@ int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
} }
int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris,
struct log_handle *logger) struct iris_table *table, struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -540,11 +562,12 @@ int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, stru
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
} }
int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, int write_intval_line(cJSON *region_json, struct iris_description *p_iris,
struct log_handle *logger) struct iris_table *table, struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -576,11 +599,12 @@ int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struc
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
} }
int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, int write_digest_line(cJSON *region_json, struct iris_description *p_iris,
struct log_handle *logger) struct iris_table *table, struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -610,11 +634,12 @@ int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struc
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
} }
int write_similar_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, int write_similar_line(cJSON *region_json, struct iris_description *p_iris,
struct log_handle *logger) struct iris_table *table, struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -640,16 +665,18 @@ int write_similar_line(cJSON *region_json, struct iris_description *p_iris, stru
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
} }
int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct iris_description *p_iris, int write_region_rule(cJSON *region_json, int compile_id, int group_id,
struct log_handle *logger) struct iris_description *p_iris, struct log_handle *logger)
{ {
cJSON *item = cJSON_GetObjectItem(region_json, "table_name"); cJSON *item = cJSON_GetObjectItem(region_json, "table_name");
if (NULL == item || item->type != cJSON_String) { if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_JSON2IRIS, "compile rule %d's table_name not defined or format error", log_error(logger, MODULE_JSON2IRIS,
compile_id); "compile rule %d's table_name not defined or format error",
compile_id);
return -1; return -1;
} }
const char *table_name = item->valuestring; const char *table_name = item->valuestring;
@@ -657,7 +684,8 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
item = cJSON_GetObjectItem(region_json, "table_type"); item = cJSON_GetObjectItem(region_json, "table_type");
if (NULL == item || item->type != cJSON_String) { if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_JSON2IRIS, log_error(logger, MODULE_JSON2IRIS,
"compile rule %d's table name %s's table_type not defined or format error", compile_id, table_name); "compile rule %d's table name %s's table_type not defined or format error",
compile_id, table_name);
return -1; return -1;
} }
@@ -665,15 +693,17 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
enum table_type table_type = TABLE_TYPE_EXPR; enum table_type table_type = TABLE_TYPE_EXPR;
int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type)); int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type));
if (ret != 1) { if (ret != 1) {
log_error(logger, MODULE_JSON2IRIS, "compile rule %d table name %s's table_type %s invalid", log_error(logger, MODULE_JSON2IRIS,
compile_id, table_name, table_type_str); "compile rule %d table name %s's table_type %s invalid",
compile_id, table_name, table_type_str);
return -1; return -1;
} }
cJSON *table_content = cJSON_GetObjectItem(region_json, "table_content"); cJSON *table_content = cJSON_GetObjectItem(region_json, "table_content");
if (NULL == table_content || table_content->type != cJSON_Object) { if (NULL == table_content || table_content->type != cJSON_Object) {
log_error(logger, MODULE_JSON2IRIS, log_error(logger, MODULE_JSON2IRIS,
"compile rule %d table name %s's table_content not defined or format error", compile_id, table_name); "compile rule %d table name %s's table_content not defined or format error",
compile_id, table_name);
return -1; return -1;
} }
@@ -709,33 +739,41 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
return ret; return ret;
} }
int write_group2compile_line(int group_id, int compile_id, int group_not_flag, int clause_index, int write_group2compile_line(int group_id, int compile_id, int group_not_flag,
const char *virtual_table, struct iris_description *p_iris) int clause_index, const char *vtable,
struct iris_description *p_iris)
{ {
char buff[4096] = {0}; char buff[4096] = {0};
struct iris_table *table = p_iris->group2compile_table; struct iris_table *table = p_iris->group2compile_table;
snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id, group_not_flag, virtual_table, clause_index); snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id,
table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); group_not_flag, vtable, clause_index);
table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz),
buff, strlen(buff));
table->line_count++; table->line_count++;
return 0; return 0;
} }
int write_group2group_line(int group_id, int superior_group_id, struct iris_description *p_iris) int write_group2group_line(int group_id, int super_group_id,
struct iris_description *p_iris)
{ {
char buff[4096] = {0}; char buff[4096] = {0};
struct iris_table *table = p_iris->group2group_table; struct iris_table *table = p_iris->group2group_table;
snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id, superior_group_id); snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id,
table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); super_group_id);
table->write_pos += memcat(&(table->buff), table->write_pos,
&(table->buff_sz), buff, strlen(buff));
table->line_count++; table->line_count++;
return 0; return 0;
} }
int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, int write_group_rule(cJSON *group_json, int parent_id,
int Nth_group, struct iris_description *p_iris, struct log_handle *logger) int parent_type, int tracking_compile_id,
int Nth_group, struct iris_description *p_iris,
struct log_handle *logger)
{ {
int ret = 0; int ret = 0;
int group_not_flag = 0; int group_not_flag = 0;
@@ -784,10 +822,11 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
if (region_json != NULL) { if (region_json != NULL) {
cJSON *region_rule = NULL; cJSON *region_rule = NULL;
cJSON_ArrayForEach(region_rule, region_json) { cJSON_ArrayForEach(region_rule, region_json) {
ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris, logger); ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id,
p_iris, logger);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_JSON2IRIS, "compile rule %d write region error", log_error(logger, MODULE_JSON2IRIS,
tracking_compile_id); "compile rule %d write region error", tracking_compile_id);
return -1; return -1;
} }
} }
@@ -799,7 +838,8 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
int i = 0; int i = 0;
cJSON_ArrayForEach(item, sub_groups) { cJSON_ArrayForEach(item, sub_groups) {
i++; i++;
ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris, logger); ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP,
tracking_compile_id, i, p_iris, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -814,20 +854,23 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
} }
if (parent_type == PARENT_TYPE_COMPILE) { if (parent_type == PARENT_TYPE_COMPILE) {
ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag, clause_index, virtual_table, p_iris); ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag,
clause_index, virtual_table, p_iris);
} else { } else {
ret = write_group2group_line(group_info->group_id, parent_id, p_iris); ret = write_group2group_line(group_info->group_id, parent_id, p_iris);
} }
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_JSON2IRIS, "%s rule %d write group error", str_parent_type[parent_type], parent_id); log_error(logger, MODULE_JSON2IRIS,
"%s rule %d write group error", str_parent_type[parent_type], parent_id);
return -1; return -1;
} }
return 0; return 0;
} }
int write_compile_line(cJSON *compile, struct iris_description *p_iris, struct log_handle *logger) int write_compile_line(cJSON *compile, struct iris_description *p_iris,
struct log_handle *logger)
{ {
cJSON *item=cJSON_GetObjectItem(compile, "compile_id"); cJSON *item=cJSON_GetObjectItem(compile, "compile_id");
if (item->type != cJSON_Number) { if (item->type != cJSON_Number) {
@@ -925,7 +968,8 @@ int write_compile_line(cJSON *compile, struct iris_description *p_iris, struct l
table_info = query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE); table_info = query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE);
} }
int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt, table_info, logger); int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt,
table_info, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -949,15 +993,18 @@ void write_table_idx(struct iris_description *p_iris, struct iris_table *table)
if (p_iris->encrypt_key) { if (p_iris->encrypt_key) {
unsigned char *encrypt_buff = NULL; unsigned char *encrypt_buff = NULL;
size_t encrypt_buff_sz = 0; size_t encrypt_buff_sz = 0;
int ret = crypt_memory(buff, table_file_sz, &encrypt_buff, &encrypt_buff_sz, p_iris->encrypt_key, int ret = crypt_memory(buff, table_file_sz, &encrypt_buff,
&encrypt_buff_sz, p_iris->encrypt_key,
p_iris->encrypt_algo, 1, err_str, sizeof(err_str)); p_iris->encrypt_algo, 1, err_str, sizeof(err_str));
assert(ret == 0); assert(ret == 0);
fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp); fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp);
fprintf(p_iris->idx_fp, "%s\t%d\t%s\t%s\n", table->table_name, table->line_count, table->table_path, p_iris->encrypt_algo); fprintf(p_iris->idx_fp, "%s\t%d\t%s\t%s\n", table->table_name, table->line_count,
table->table_path, p_iris->encrypt_algo);
FREE(encrypt_buff); FREE(encrypt_buff);
} else { } else {
fwrite(buff, table_file_sz, 1, table_fp); fwrite(buff, table_file_sz, 1, table_fp);
fprintf(p_iris->idx_fp, "%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); fprintf(p_iris->idx_fp, "%s\t%d\t%s\n", table->table_name, table->line_count,
table->table_path);
} }
fclose(table_fp); fclose(table_fp);
@@ -968,8 +1015,9 @@ int write_index_file(struct iris_description *p_iris, struct log_handle *logger)
{ {
p_iris->idx_fp = fopen(p_iris->index_path, "w"); p_iris->idx_fp = fopen(p_iris->index_path, "w");
if (NULL == p_iris->idx_fp) { if (NULL == p_iris->idx_fp) {
log_error(logger, MODULE_JSON2IRIS, "index file %s fopen error %s", log_error(logger, MODULE_JSON2IRIS,
p_iris->index_path, strerror(errno)); "index file %s fopen error %s",
p_iris->index_path, strerror(errno));
return -1; return -1;
} }
@@ -985,7 +1033,8 @@ int write_index_file(struct iris_description *p_iris, struct log_handle *logger)
return 0; return 0;
} }
int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *logger) int write_iris(cJSON *json, struct iris_description *p_iris,
struct log_handle *logger)
{ {
int i=0; int i=0;
int ret=0; int ret=0;
@@ -1017,7 +1066,8 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *
parent_group = group_info_add_unsafe(p_iris, parent_group_name); parent_group = group_info_add_unsafe(p_iris, parent_group_name);
} }
ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris, logger); ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP,
0, 0, p_iris, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -1041,20 +1091,23 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *
group_array = cJSON_GetObjectItem(compile_obj, "groups"); group_array = cJSON_GetObjectItem(compile_obj, "groups");
if (NULL == group_array) { if (NULL == group_array) {
log_error(logger, MODULE_JSON2IRIS, "compile rule %d have no group", compile_id); log_error(logger, MODULE_JSON2IRIS,
"compile rule %d have no group", compile_id);
return -1; return -1;
} }
int group_cnt = cJSON_GetArraySize(group_array); int group_cnt = cJSON_GetArraySize(group_array);
if (group_cnt <= 0) { if (group_cnt <= 0) {
log_error(logger, MODULE_JSON2IRIS, "compile rule %d have no groups", compile_id); log_error(logger, MODULE_JSON2IRIS,
"compile rule %d have no groups", compile_id);
return -1; return -1;
} }
i = 0; i = 0;
cJSON *group_obj = NULL; cJSON *group_obj = NULL;
cJSON_ArrayForEach(group_obj, group_array) { cJSON_ArrayForEach(group_obj, group_array) {
ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, compile_id, i, p_iris, logger); ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE,
compile_id, i, p_iris, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -1070,8 +1123,9 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *
return 0; return 0;
} }
int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx, int json2iris(const char *json_buff, const char *json_filename,
char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo, redisContext *redis_write_ctx, char *iris_dir_buf,
int buf_len, char *encrypt_key, char *encrypt_algo,
struct log_handle *logger) struct log_handle *logger)
{ {
int ret = -1; int ret = -1;
@@ -1085,7 +1139,8 @@ int json2iris(const char *json_buff, const char *json_filename, redisContext *re
cJSON *json = cJSON_Parse(json_buff); cJSON *json = cJSON_Parse(json_buff);
if (!json) { if (!json) {
log_error(logger, MODULE_JSON2IRIS, "Error before: %-200.200s", cJSON_GetErrorPtr()); log_error(logger, MODULE_JSON2IRIS,
"Error before: %-200.200s", cJSON_GetErrorPtr());
goto error_out; goto error_out;
} }
@@ -1105,15 +1160,16 @@ int json2iris(const char *json_buff, const char *json_filename, redisContext *re
} }
ret = set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, ret = set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo,
compile_tbl_name, group2compile_tbl_name, group2group_tbl_name, compile_tbl_name, group2compile_tbl_name,
redis_write_ctx, &iris_cfg); group2group_tbl_name, redis_write_ctx, &iris_cfg);
if (ret < 0) { if (ret < 0) {
goto error_out; goto error_out;
} }
ret = create_tmp_dir(&iris_cfg); ret = create_tmp_dir(&iris_cfg);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_JSON2IRIS, "create tmp folder %s error", iris_cfg.tmp_iris_dir); log_error(logger, MODULE_JSON2IRIS,
"create tmp folder %s error", iris_cfg.tmp_iris_dir);
goto error_out; goto error_out;
} }
@@ -1121,7 +1177,8 @@ int json2iris(const char *json_buff, const char *json_filename, redisContext *re
if (ret < 0) { if (ret < 0) {
goto error_out; goto error_out;
} }
memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir) + 1, (unsigned int)buf_len)); memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir,
MIN(strlen(iris_cfg.tmp_iris_index_dir) + 1, (unsigned int)buf_len));
cJSON_Delete(json); cJSON_Delete(json);
clear_iris_descriptor(&iris_cfg); clear_iris_descriptor(&iris_cfg);

467
src/maat_api.cpp
View File

@@ -12,7 +12,6 @@
#include <string.h> #include <string.h>
#include <assert.h> #include <assert.h>
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "json2iris.h" #include "json2iris.h"
#include "maat/maat.h" #include "maat/maat.h"
@@ -38,15 +37,10 @@
#define DISTRICT_ANY -1 #define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2 #define DISTRICT_UNKNOWN -2
inline int scan_state_should_compile_NOT(struct maat_state *mid) enum district_set_flag {
{ DISTRICT_FLAG_UNSET,
if (mid && mid->is_last_scan==1 && mid->compile_mid && DISTRICT_FLAG_SET
maat_compile_state_has_NOT_clause(mid->compile_mid)) { };
return 1;
} else {
return 0;
}
}
struct maat_options* maat_options_new(void) struct maat_options* maat_options_new(void)
{ {
@@ -111,7 +105,8 @@ int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms)
return 0; return 0;
} }
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len) int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name,
size_t name_len)
{ {
memcpy(opts->instance_name, instance_name, name_len); memcpy(opts->instance_name, instance_name, name_len);
@@ -125,7 +120,8 @@ int maat_options_set_deferred_load_on(struct maat_options *opts)
return 0; return 0;
} }
int maat_options_set_iris(struct maat_options *opts, const char *full_directory, const char *increment_directory) int maat_options_set_iris(struct maat_options *opts, const char *full_directory,
const char *increment_directory)
{ {
if (strlen(full_directory) >= NAME_MAX || strlen(increment_directory) >= NAME_MAX) { if (strlen(full_directory) >= NAME_MAX || strlen(increment_directory) >= NAME_MAX) {
return -1; return -1;
@@ -146,7 +142,8 @@ int maat_options_set_json_file(struct maat_options *opts, const char *json_filen
return 0; return 0;
} }
int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, uint16_t redis_port, int redis_db) int maat_options_set_redis(struct maat_options *opts, const char *redis_ip,
uint16_t redis_port, int redis_db)
{ {
memcpy(opts->redis_ctx.redis_ip, redis_ip, strlen(redis_ip)); memcpy(opts->redis_ctx.redis_ip, redis_ip, strlen(redis_ip));
opts->redis_ctx.redis_port = redis_port; opts->redis_ctx.redis_port = redis_port;
@@ -172,10 +169,13 @@ void maat_read_full_config(struct maat *maat_instance)
switch (maat_instance->input_mode) { switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS: case DATA_SOURCE_REDIS:
mr_ctx = &(maat_instance->mr_ctx); mr_ctx = &(maat_instance->mr_ctx);
log_info(maat_instance->logger, MODULE_MAAT_API, "Maat initiate from Redis %s:%hu db%d", log_info(maat_instance->logger, MODULE_MAAT_API,
"Maat initiate from Redis %s:%hu db%d",
mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip,
mr_ctx->redis_db, maat_instance->logger); mr_ctx->redis_port,
mr_ctx->redis_db,
maat_instance->logger);
if (mr_ctx->read_ctx != NULL) { if (mr_ctx->read_ctx != NULL) {
redis_monitor_traverse(maat_instance->maat_version, mr_ctx, redis_monitor_traverse(maat_instance->maat_version, mr_ctx,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
@@ -200,9 +200,11 @@ void maat_read_full_config(struct maat *maat_instance)
} }
break; break;
case DATA_SOURCE_JSON_FILE: case DATA_SOURCE_JSON_FILE:
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str)); ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file,
err_str, sizeof(err_str));
if (ret < 0) { if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_API, "Maat re-initiate with JSON file %s failed: %s", log_error(maat_instance->logger, MODULE_MAAT_API,
"Maat re-initiate with JSON file %s failed: %s",
maat_instance->json_ctx.json_file, err_str); maat_instance->json_ctx.json_file, err_str);
return; return;
} }
@@ -212,7 +214,8 @@ void maat_read_full_config(struct maat *maat_instance)
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance, maat_instance->logger); maat_instance, maat_instance->logger);
if (NULL == maat_instance->creating_maat_rt) { if (NULL == maat_instance->creating_maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_API, "At initiation: NO effective rule in %s", log_error(maat_instance->logger, MODULE_MAAT_API,
"At initiation: NO effective rule in %s",
maat_instance->json_ctx.iris_file); maat_instance->json_ctx.iris_file);
} }
break; break;
@@ -243,36 +246,44 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
} else { } else {
char log_path[1024] = {0}; char log_path[1024] = {0};
if (strlen(maat_instance->instance_name) > 0) { if (strlen(maat_instance->instance_name) > 0) {
snprintf(log_path, sizeof(log_path), "%s.log", maat_instance->instance_name); snprintf(log_path, sizeof(log_path), "%s.log",
maat_instance->instance_name);
} else { } else {
snprintf(log_path, sizeof(log_path), "maat.log"); snprintf(log_path, sizeof(log_path), "maat.log");
} }
maat_instance->logger = log_handle_create(log_path, opts->log_level); maat_instance->logger = log_handle_create(log_path, opts->log_level);
} }
maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags, maat_instance->logger); maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags,
maat_instance->garbage_bin, maat_instance->logger);
if (NULL == maat_instance->tbl_mgr) { if (NULL == maat_instance->tbl_mgr) {
goto failed; goto failed;
} }
maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr); maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
maat_instance->input_mode = opts->input_mode; maat_instance->input_mode = opts->input_mode;
switch (maat_instance->input_mode) { switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS: case DATA_SOURCE_REDIS:
memcpy(maat_instance->mr_ctx.redis_ip, opts->redis_ctx.redis_ip, strlen(opts->redis_ctx.redis_ip)); memcpy(maat_instance->mr_ctx.redis_ip, opts->redis_ctx.redis_ip,
strlen(opts->redis_ctx.redis_ip));
maat_instance->mr_ctx.redis_port = opts->redis_ctx.redis_port; maat_instance->mr_ctx.redis_port = opts->redis_ctx.redis_port;
maat_instance->mr_ctx.redis_db = opts->redis_ctx.redis_db; maat_instance->mr_ctx.redis_db = opts->redis_ctx.redis_db;
break; break;
case DATA_SOURCE_IRIS_FILE: case DATA_SOURCE_IRIS_FILE:
memcpy(maat_instance->iris_ctx.full_idx_dir, opts->iris_ctx.full_idx_dir, strlen(opts->iris_ctx.full_idx_dir)); memcpy(maat_instance->iris_ctx.full_idx_dir, opts->iris_ctx.full_idx_dir,
memcpy(maat_instance->iris_ctx.inc_idx_dir, opts->iris_ctx.inc_idx_dir, strlen(opts->iris_ctx.inc_idx_dir)); strlen(opts->iris_ctx.full_idx_dir));
memcpy(maat_instance->iris_ctx.inc_idx_dir, opts->iris_ctx.inc_idx_dir,
strlen(opts->iris_ctx.inc_idx_dir));
break; break;
case DATA_SOURCE_JSON_FILE: case DATA_SOURCE_JSON_FILE:
memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file)); memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file,
strlen(opts->json_ctx.json_file));
break; break;
default: default:
log_error(maat_instance->logger, MODULE_MAAT_API, "data source unsupported:%d", maat_instance->input_mode); log_error(maat_instance->logger, MODULE_MAAT_API,
"data source unsupported:%d", maat_instance->input_mode);
goto failed; goto failed;
} }
@@ -287,9 +298,11 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
(maat_instance->gc_timeout_ms / 1000); (maat_instance->gc_timeout_ms / 1000);
maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s); maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s);
maat_instance->outer_mid_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); maat_instance->outer_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->compile_mid_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); maat_instance->compile_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->hit_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->not_grp_hit_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
pthread_mutex_init(&(maat_instance->background_update_mutex), NULL); pthread_mutex_init(&(maat_instance->background_update_mutex), NULL);
@@ -349,7 +362,8 @@ size_t generic_plugin_runtime_cached_row_count(void *custom_rt, enum table_type
return 0; return 0;
} }
const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type, size_t row_id) const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type,
size_t row_id)
{ {
return NULL; return NULL;
} }
@@ -383,7 +397,7 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id,
if (row_cnt > 0) { if (row_cnt > 0) {
if (start != NULL) { if (start != NULL) {
start(MAAT_RULE_UPDATE_TYPE_FULL, u_para); start(MAAT_UPDATE_TYPE_FULL, u_para);
} }
for (size_t i = 0; i < row_cnt; i++) { for (size_t i = 0; i < row_cnt; i++) {
@@ -457,8 +471,9 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t
return 0; return 0;
} }
void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, int table_id, void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema,
enum table_type table_type, int valid_column) int table_id, enum table_type table_type,
int valid_column)
{ {
struct ex_data_schema *ex_data_schema = NULL; struct ex_data_schema *ex_data_schema = NULL;
struct ex_data_runtime *ex_data_rt = NULL; struct ex_data_runtime *ex_data_rt = NULL;
@@ -518,26 +533,32 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i
{ {
pthread_mutex_lock(&(maat_instance->background_update_mutex)); pthread_mutex_lock(&(maat_instance->background_update_mutex));
int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id, int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id,
new_func, free_func, dup_func, argl, argp, new_func, free_func, dup_func,
maat_instance->logger); argl, argp, maat_instance->logger);
if (ret < 0) { if (ret < 0) {
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return -1; return -1;
} }
enum table_type table_type = TABLE_TYPE_INVALID;
int valid_column = -1;
if (maat_instance->maat_rt != NULL) { if (maat_instance->maat_rt != NULL) {
void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); assert(runtime != NULL && schema != NULL);
int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id);
generic_plugin_runtime_commit_ex_schema(runtime, schema, table_id, table_type, valid_column); table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id);
generic_plugin_runtime_commit_ex_schema(runtime, schema, table_id, table_type,
valid_column);
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return 0; return 0;
} }
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, const char *key, size_t key_len) void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
const char *key, size_t key_len)
{ {
struct maat_runtime *maat_rt = maat_instance->maat_rt; struct maat_runtime *maat_rt = maat_instance->maat_rt;
if (NULL == maat_rt) { if (NULL == maat_rt) {
@@ -569,11 +590,6 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, co
return ex_data_runtime_get_ex_data(ex_data_rt, key, key_len); return ex_data_runtime_get_ex_data(ex_data_rt, key, key_len);
} }
static void scan_count_inc(struct maat_state *mid)
{
mid->scan_cnt++;
}
struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id) struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id)
{ {
struct maat_state *outer_state = NULL; struct maat_state *outer_state = NULL;
@@ -587,7 +603,8 @@ struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id)
return outer_state; return outer_state;
} }
struct maat_state *grab_state(struct maat_state **state, struct maat *maat_instance, int thread_id) struct maat_state *grab_state(struct maat_state **state, struct maat *maat_instance,
int thread_id)
{ {
struct maat_state *mid = *state; struct maat_state *mid = *state;
@@ -596,46 +613,51 @@ struct maat_state *grab_state(struct maat_state **state, struct maat *maat_insta
*state = mid; *state = mid;
//Maat_set_scan_status calls grap_mid() with thread_num=-1. //Maat_set_scan_status calls grap_mid() with thread_num=-1.
if (mid->thread_id >= 0) { if (mid->thread_id >= 0) {
alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1); alignment_int64_array_add(maat_instance->outer_state_cnt, thread_id, 1);
} }
} }
if (mid->thread_id < 0 && thread_id >= 0) { if (mid->thread_id < 0 && thread_id >= 0) {
mid->thread_id = thread_id; mid->thread_id = thread_id;
alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1); alignment_int64_array_add(maat_instance->outer_state_cnt, thread_id, 1);
} }
if (NULL == mid->compile_mid) { if (NULL == mid->compile_state) {
mid->compile_mid = maat_compile_state_new(thread_id); mid->compile_state = maat_compile_state_new(thread_id);
alignment_int64_array_add(maat_instance->compile_mid_cnt, thread_id, 1); alignment_int64_array_add(maat_instance->compile_state_cnt, thread_id, 1);
} }
return mid; return mid;
} }
int hit_group_to_compile(void *g2g_runtime, int *group_ids, int group_hit_cnt, int vt_id, inline int scan_status_should_compile_NOT(struct maat_state *state)
void *compile_runtime, int *compile_ids, size_t n_compile_id, {
if (state && (LAST_SCAN_SET == state->is_last_scan) && state->compile_state
&& maat_compile_state_has_NOT_clause(state->compile_state)) {
return 1;
}
return 0;
}
int hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t compile_ids_size,
size_t *n_hit_compile_id, struct maat_state *mid) size_t *n_hit_compile_id, struct maat_state *mid)
{ {
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; int compile_id_cnt = compile_runtime_match((struct compile_runtime *)compile_runtime,
int n_top_group_ids = group2group_runtime_get_top_groups(g2g_runtime, group_ids, group_hit_cnt, top_group_ids); compile_ids, compile_ids_size, mid);
size_t n_all_group_ids = 0; *n_hit_compile_id = compile_id_cnt;
if (n_top_group_ids > 0) { if (compile_id_cnt > 0) {
n_all_group_ids = group_hit_cnt + n_top_group_ids; return MAAT_SCAN_HIT;
} else { } else {
n_all_group_ids = group_hit_cnt; return MAAT_SCAN_HALF_HIT;
} }
}
int all_group_ids[n_all_group_ids] = {-1}; int maat_scan_flag(struct maat *instance, int table_id, int thread_id,
// maat state find compile_table_id, if not found, maat_instance->default_compile_table_id uint64_t flag, int *results, size_t n_result,
int n_compile_ids = compile_runtime_match((struct compile_runtime *)compile_runtime, all_group_ids, n_all_group_ids, size_t *n_hit_result, struct maat_state **state)
vt_id, compile_ids, n_compile_id, mid); {
*n_hit_compile_id = n_compile_ids; return 0;
if (n_compile_ids > 0) {
return MAAT_HIT;
} else {
return MAAT_HALF_HIT;
}
} }
int maat_scan_integer(struct maat *instance, int table_id, int thread_id, int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
@@ -651,18 +673,20 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
{ {
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) { (thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) {
return -1; return MAAT_SCAN_ERR;
} }
struct maat_state *mid = NULL; struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, thread_id); mid = grab_state(state, maat_instance, thread_id);
scan_count_inc(mid); mid->scan_cnt++;
int virtual_table_id = 0; maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
int vtable_id = 0;
// struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
// SCAN_TYPE_IP, &virtual_table_id); // SCAN_TYPE_IP, &virtual_table_id);
// if (NULL == table_schema) { // if (NULL == table_schema) {
// return MAAT_ERR; // return MAAT_SCAN_ERR;
// } // }
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
@@ -670,74 +694,243 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
assert(table_type == TABLE_TYPE_IP_PLUS); assert(table_type == TABLE_TYPE_IP_PLUS);
// int table_id = table_schema_get_table_id(real_table); // int table_id = table_schema_get_table_id(real_table);
void *ip_plus_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
if (NULL == ip_rt) {
return MAAT_SCAN_ERR;
}
// size_t rule_num = table_runtime_rule_count(table_rt); // size_t rule_num = table_runtime_rule_count(table_rt);
// if (0 == rule_num) { // if (0 == rule_num) {
// return 0; // return 0;
// } // }
int group_hit_cnt = ip_plus_runtime_scan_ipv4((struct ip_plus_runtime *)ip_plus_rt, thread_id, ip_addr, int group_hit_cnt = ip_runtime_scan_ip((struct ip_runtime *)ip_rt, thread_id, IPv4,
group_ids, MAX_SCANNER_HIT_GROUP_NUM, virtual_table_id, mid); (uint8_t *)&ip_addr, group_ids, sizeof(group_ids),
vtable_id, mid);
if (group_hit_cnt < 0) { if (group_hit_cnt < 0) {
return MAAT_ERR; return MAAT_SCAN_ERR;
} else if (0 == group_hit_cnt) {
return MAAT_OK;
} else {
// come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
void *compile_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id);
int ret = hit_group_to_compile(g2g_runtime, group_ids, group_hit_cnt, virtual_table_id, compile_runtime,
results, n_result, n_hit_result, mid);
return ret;
} }
int compile_ret = 0;
if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
if (group_hit_cnt > 0) {
ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
}
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;
} else {
compile_table_id = mid->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid);
assert(mid->is_last_scan < LAST_SCAN_FINISHED);
if (LAST_SCAN_SET == mid->is_last_scan) {
mid->is_last_scan = LAST_SCAN_FINISHED;
}
}
if (compile_ret > 0) {
alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
}
if (0 == group_hit_cnt && compile_ret > 0) {
// hit NOT group
alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
}
maat_runtime_ref_dec(maat_instance->maat_rt, thread_id);
if (0 == compile_ret && group_hit_cnt > 0) {
return MAAT_SCAN_HALF_HIT;
}
return MAAT_SCAN_HIT;
} }
int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id, int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id,
uint8_t *ip_addr, int *results, size_t n_result, uint8_t *ip_addr, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state) size_t *n_hit_result, struct maat_state **state)
{ {
return 0; if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == ip_addr) || (NULL == results) || (0 == n_result) ||
(NULL == state)) {
return MAAT_SCAN_ERR;
}
struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, thread_id);
mid->scan_cnt++;
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
int vtable_id = 0;
// struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
// SCAN_TYPE_IP, &virtual_table_id);
// if (NULL == table_schema) {
// return MAAT_SCAN_ERR;
// }
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
assert(table_type == TABLE_TYPE_IP_PLUS);
// int table_id = table_schema_get_table_id(real_table);
void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
if (NULL == ip_rt) {
return MAAT_SCAN_ERR;
}
// size_t rule_num = table_runtime_rule_count(table_rt);
// if (0 == rule_num) {
// return 0;
// }
int group_hit_cnt = ip_runtime_scan_ip((struct ip_runtime *)ip_rt, thread_id, IPv6, ip_addr,
group_ids, sizeof(group_ids), vtable_id, mid);
if (group_hit_cnt < 0) {
return MAAT_SCAN_ERR;
}
int compile_ret = 0;
if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
if (group_hit_cnt > 0) {
ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
}
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;
} else {
compile_table_id = mid->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid);
assert(mid->is_last_scan < LAST_SCAN_FINISHED);
if (LAST_SCAN_SET == mid->is_last_scan) {
mid->is_last_scan = LAST_SCAN_FINISHED;
}
}
if (compile_ret > 0) {
alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
}
if (0 == group_hit_cnt && compile_ret > 0) {
// hit NOT group
alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
}
maat_runtime_ref_dec(maat_instance->maat_rt, thread_id);
if (0 == compile_ret && group_hit_cnt > 0) {
return MAAT_SCAN_HALF_HIT;
}
return MAAT_SCAN_HIT;
} }
int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
const char *data, size_t data_len, int *results, size_t n_result, const char *data, size_t data_len, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state) size_t *n_hit_result, struct maat_state **state)
{ {
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM)
(thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) || || (thread_id < 0) || (NULL == data) || (0 == data_len)
(0 == n_result) || (NULL == state)) { || (NULL == results) || (0 == n_result) || (NULL == state)) {
return MAAT_ERR; return MAAT_SCAN_ERR;
} }
struct maat_state *mid = NULL; struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, thread_id); mid = grab_state(state, maat_instance, thread_id);
scan_count_inc(mid); mid->scan_cnt++;
int virtual_table_id = 0; int vtable_id = 0;
//TODO: by luis get virtual_table_id //TODO: by luis get virtual_table_id
// struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
// SCAN_TYPE_STRING, &virtual_table_id); // SCAN_TYPE_STRING, &virtual_table_id);
// if (NULL == table_schema) { // if (NULL == table_schema) {
// return MAAT_ERR; // return MAAT_SCAN_ERR;
// } // }
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; if (NULL == maat_instance->maat_rt) {
void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); log_error(maat_instance->logger, MODULE_MAAT_API,
int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)runtime, thread_id, data, data_len, "maat_scan_string error because of maat_runtime is NULL");
group_ids, MAX_SCANNER_HIT_GROUP_NUM, virtual_table_id, mid); return MAAT_SCAN_OK;
if (group_hit_cnt < 0) {
return MAAT_ERR;
} else if (0 == group_hit_cnt) {
return MAAT_OK;
} else {
// come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
void *compile_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id);
int ret = hit_group_to_compile(g2g_runtime, group_ids, group_hit_cnt, virtual_table_id, compile_runtime,
results, n_result, n_hit_result, mid);
return ret;
} }
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
//TODO: is TABLE_TYPE_EXPR_PLUS
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
maat_instance->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
void *expr_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
if (NULL == expr_rt) {
return MAAT_SCAN_ERR;
}
int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, thread_id,
data, data_len, group_ids, sizeof(group_ids),
vtable_id, mid);
if (group_hit_cnt < 0) {
return MAAT_SCAN_ERR;
}
int compile_ret = 0;
if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
}
if (group_hit_cnt > 0 && table_type == TABLE_TYPE_EXPR_PLUS) {
}
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;
} else {
compile_table_id = mid->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid);
assert(mid->is_last_scan < LAST_SCAN_FINISHED);
if (LAST_SCAN_SET == mid->is_last_scan) {
mid->is_last_scan = LAST_SCAN_FINISHED;
}
}
if (compile_ret > 0) {
alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
}
if (0 == group_hit_cnt && compile_ret > 0) {
// hit NOT group
alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
}
maat_runtime_ref_dec(maat_instance->maat_rt, thread_id);
if (0 == compile_ret && group_hit_cnt > 0) {
return MAAT_SCAN_HALF_HIT;
}
return MAAT_SCAN_HIT;
} }
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id) struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id)
@@ -745,8 +938,9 @@ struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, i
return NULL; return NULL;
} }
int maat_scan_stream(struct maat_stream **stream, int thread_id, const char *data, int data_len, int maat_scan_stream(struct maat_stream **stream, int thread_id,
int results[], size_t *n_result, struct maat_state **state) const char *data, int data_len, int results[],
size_t *n_result, struct maat_state **state)
{ {
return 0; return 0;
} }
@@ -756,19 +950,27 @@ void maat_scan_stream_close(struct maat_stream **stream)
} }
int maat_state_set_scan_district(struct maat *maat_instance, struct maat_state **state, const char *district, size_t district_len) int maat_state_set_scan_district(struct maat *maat_instance,
struct maat_state **state,
const char *district,
size_t district_len)
{ {
if (NULL == maat_instance->maat_rt || NULL == district || district_len <= 0) { if (NULL == maat_instance->maat_rt || NULL == district ||
district_len <= 0) {
return -1; return -1;
} }
struct maat_state *mid = grab_state(state, maat_instance, -1); struct maat_state *mid = grab_state(state, maat_instance, -1);
int map_ret = maat_kv_read_unNull(maat_instance->maat_rt->district_map, district, district_len, &(mid->district_id)); int ret = table_manager_set_scan_district(maat_instance->tbl_mgr,
if (map_ret < 0) { district, district_len,
&(mid->district_id));
// int map_ret = maat_kv_read_unNull(maat_instance->maat_rt->district_map,
// district, district_len, &(mid->district_id));
if (ret < 0) {
mid->district_id = DISTRICT_UNKNOWN; mid->district_id = DISTRICT_UNKNOWN;
} }
mid->is_set_district = 1; mid->is_set_district = DISTRICT_FLAG_SET;
return 0; return 0;
} }
@@ -780,8 +982,8 @@ int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **sta
} }
struct maat_state *mid = grab_state(state, maat_instance, -1); struct maat_state *mid = grab_state(state, maat_instance, -1);
assert(mid->is_last_scan == 0); assert(mid->is_last_scan == LAST_SCAN_UNSET);
mid->is_last_scan = 1; mid->is_last_scan = LAST_SCAN_SET;
return 0; return 0;
} }
@@ -799,26 +1001,27 @@ int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_st
return 0; return 0;
} }
size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid, size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *state,
struct maat_hit_path *paths, size_t n_path) struct maat_hit_path *paths, size_t n_path)
{ {
int compile_table_id = -1; int compile_table_id = -1;
if (mid->compile_table_id == -1) { if (state->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id; compile_table_id = maat_instance->default_compile_table_id;
} else { } else {
compile_table_id = mid->compile_table_id; compile_table_id = state->compile_table_id;
} }
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, assert(NULL != compile_rt && NULL != g2g_runtime);
(struct group2group_runtime *)g2g_runtime, mid->compile_mid, paths, n_path); return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt,
(struct group2group_runtime *)g2g_runtime,
return 0; state->compile_state, paths, n_path);
} }
size_t maat_get_hit_objects(struct maat_compile_state *compile_state, struct maat_hit_object *objs, size_t n_objs) size_t maat_get_hit_objects(struct maat_compile_state *compile_state,
struct maat_hit_object *objs, size_t n_objs)
{ {
return 0; return 0;
} }
@@ -829,7 +1032,7 @@ int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **sta
struct maat_state *mid = NULL; struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, 0); mid = grab_state(state, maat_instance, 0);
if (NULL == mid->compile_mid || NULL == maat_instance->maat_rt) { if (NULL == mid->compile_state || NULL == maat_instance->maat_rt) {
return 0; return 0;
} }
@@ -853,13 +1056,13 @@ void maat_state_free(struct maat_state **state)
mid = *state; mid = *state;
if (mid->thread_id >= 0) { if (mid->thread_id >= 0) {
alignment_int64_array_add(mid->maat_instance->outer_mid_cnt, mid->thread_id, -1); alignment_int64_array_add(mid->maat_instance->outer_state_cnt, mid->thread_id, -1);
} }
if (mid->compile_mid != NULL) { if (mid->compile_state != NULL) {
maat_compile_state_free(mid->compile_mid); maat_compile_state_free(mid->compile_state);
mid->compile_mid = NULL; mid->compile_state = NULL;
alignment_int64_array_add(mid->maat_instance->compile_mid_cnt, mid->thread_id, -1); alignment_int64_array_add(mid->maat_instance->compile_state_cnt, mid->thread_id, -1);
} }
mid->maat_instance = NULL; mid->maat_instance = NULL;

31
src/maat_command.cpp
View File

@@ -13,7 +13,6 @@
#include <unistd.h> #include <unistd.h>
#include <string.h> #include <string.h>
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_command.h" #include "maat_command.h"
#include "maat_rule.h" #include "maat_rule.h"
@@ -52,7 +51,8 @@ redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...
return (redisReply *)reply; return (redisReply *)reply;
} }
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db, struct log_handle *logger) redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger)
{ {
struct timeval connect_timeout; struct timeval connect_timeout;
connect_timeout.tv_sec = 0; connect_timeout.tv_sec = 0;
@@ -116,10 +116,12 @@ void maat_cmd_clear_rule_cache(struct serial_rule *s_rule)
memset(s_rule, 0, sizeof(struct serial_rule)); memset(s_rule, 0, sizeof(struct serial_rule));
} }
int connect_redis_for_write(struct source_redis_ctx *mr_ctx, struct log_handle *logger) int connect_redis_for_write(struct source_redis_ctx *mr_ctx,
struct log_handle *logger)
{ {
assert(mr_ctx->write_ctx == NULL); assert(mr_ctx->write_ctx == NULL);
mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, logger); mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port,
mr_ctx->redis_db, logger);
if (NULL == mr_ctx->write_ctx) { if (NULL == mr_ctx->write_ctx) {
return -1; return -1;
} else { } else {
@@ -130,7 +132,8 @@ int connect_redis_for_write(struct source_redis_ctx *mr_ctx, struct log_handle *
redisContext *get_redis_ctx_for_write(struct maat *maat_instance) redisContext *get_redis_ctx_for_write(struct maat *maat_instance)
{ {
if (NULL == maat_instance->mr_ctx.write_ctx) { if (NULL == maat_instance->mr_ctx.write_ctx) {
int ret = connect_redis_for_write(&(maat_instance->mr_ctx), maat_instance->logger); int ret = connect_redis_for_write(&(maat_instance->mr_ctx),
maat_instance->logger);
if(ret!=0) if(ret!=0)
{ {
return NULL; return NULL;
@@ -139,8 +142,9 @@ redisContext *get_redis_ctx_for_write(struct maat *maat_instance)
return maat_instance->mr_ctx.write_ctx; return maat_instance->mr_ctx.write_ctx;
} }
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id, void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
const char *table_name, const char *line, long long timeout) unsigned long rule_id, const char *table_name,
const char *line, long long timeout)
{ {
memset(rule, 0, sizeof(struct serial_rule)); memset(rule, 0, sizeof(struct serial_rule));
rule->op = op; rule->op = op;
@@ -153,7 +157,8 @@ void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
} }
} }
int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq) int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
int valid_column_seq)
{ {
int column_seq = 0; int column_seq = 0;
@@ -204,7 +209,8 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
size_t len = 0; size_t len = 0;
int ret = get_column_pos(line, column_seq, &offset, &len); int ret = get_column_pos(line, column_seq, &offset, &len);
// 0 is also a valid value for some non-MAAT producer. // 0 is also a valid value for some non-MAAT producer.
if (ret < 0 || offset >= strlen(line) || (line[offset] != '1' && line[offset] != '0')) { if (ret < 0 || offset >= strlen(line) || (line[offset] != '1' &&
line[offset] != '0')) {
return -1; return -1;
} }
@@ -306,7 +312,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, line_rule->table_name); int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, line_rule->table_name);
if (table_id < 0) { if (table_id < 0) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND, "Command set line id %d failed: unknown table %s", log_error(maat_instance->logger, MODULE_MAAT_COMMAND,
"Command set line id %d failed: unknown table %s",
line_rule->rule_id, line_rule->table_name); line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);
return -1; return -1;
@@ -328,8 +335,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
absolute_expire_time = server_time + line_rule->expire_after; absolute_expire_time = server_time + line_rule->expire_after;
} }
maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name, maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id,
line_rule->table_line, absolute_expire_time); line_rule->table_name, line_rule->table_line, absolute_expire_time);
int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time, maat_instance->logger); int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time, maat_instance->logger);
if (success_cnt != 1) { if (success_cnt != 1) {

481
src/maat_compile.cpp
View File

File diff suppressed because it is too large Load Diff

44
src/maat_config_monitor.cpp
View File

@@ -16,7 +16,6 @@
#include <dirent.h> #include <dirent.h>
#include <unistd.h> #include <unistd.h>
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "json2iris.h" #include "json2iris.h"
@@ -84,7 +83,8 @@ const char *path2filename(const char *path)
return path + i + 1; return path + i + 1;
} }
char *read_nxt_line_from_buff(const char *buff, size_t buff_size, size_t *offset, char *line, int line_size) char *read_nxt_line_from_buff(const char *buff, size_t buff_size,
size_t *offset, char *line, int line_size)
{ {
int this_offset=0; int this_offset=0;
const char* p; const char* p;
@@ -221,7 +221,7 @@ int get_new_idx_path(long long current_version, const char *file_dir,
char ***idx_path, size_t *idx_num, struct log_handle *logger) char ***idx_path, size_t *idx_num, struct log_handle *logger)
{ {
struct dirent **namelist = NULL; struct dirent **namelist = NULL;
int update_type = CM_UPDATE_TYPE_NONE; int update_type = MAAT_UPDATE_TYPE_NONE;
int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort); int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort);
if (n < 0) { if (n < 0) {
@@ -285,7 +285,7 @@ int get_new_idx_path(long long current_version, const char *file_dir,
(*idx_path)[0] = (char *)malloc(path_len); (*idx_path)[0] = (char *)malloc(path_len);
snprintf((*idx_path)[0], path_len, "%s/%s", file_dir, namelist[full_file_idx]->d_name); snprintf((*idx_path)[0], path_len, "%s/%s", file_dir, namelist[full_file_idx]->d_name);
*idx_num = 1; *idx_num = 1;
update_type = CM_UPDATE_TYPE_FULL; update_type = MAAT_UPDATE_TYPE_FULL;
} else if (latest_inc_version > current_version) { } else if (latest_inc_version > current_version) {
//inc update,it's possible that do inc after full update in this function,but we'll process it at next loop. //inc update,it's possible that do inc after full update in this function,but we'll process it at next loop.
*idx_path = (char **)malloc(sizeof(char **) * inc_idx_num); *idx_path = (char **)malloc(sizeof(char **) * inc_idx_num);
@@ -295,9 +295,9 @@ int get_new_idx_path(long long current_version, const char *file_dir,
snprintf((*idx_path)[i], path_len, "%s/%s", file_dir, namelist[inc_file_idx[i]]->d_name); snprintf((*idx_path)[i], path_len, "%s/%s", file_dir, namelist[inc_file_idx[i]]->d_name);
} }
*idx_num = inc_idx_num; *idx_num = inc_idx_num;
update_type = CM_UPDATE_TYPE_INC; update_type = MAAT_UPDATE_TYPE_INC;
} else { } else {
update_type = CM_UPDATE_TYPE_NONE; update_type = MAAT_UPDATE_TYPE_NONE;
} }
FREE(inc_file_idx); FREE(inc_file_idx);
@@ -324,7 +324,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
memset(table_array, 0, sizeof(table_array)); memset(table_array, 0, sizeof(table_array));
int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num, logger); int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num, logger);
if (update_type != CM_UPDATE_TYPE_NONE) { if (update_type != MAAT_UPDATE_TYPE_NONE) {
for (i = 0; i < idx_path_num; i++) { for (i = 0; i < idx_path_num; i++) {
log_info(logger, MODULE_CONFIG_MONITOR, "load %s", idx_path_array[i]); log_info(logger, MODULE_CONFIG_MONITOR, "load %s", idx_path_array[i]);
int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM); int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM);
@@ -370,12 +370,15 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
size_t uncompressed_buff_sz = 0; size_t uncompressed_buff_sz = 0;
log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, log_info(maat_instance->logger, MODULE_CONFIG_MONITOR,
"Maat initial with JSON file %s, formating...", json_filename); "Maat initial with JSON file %s, formating...",
json_filename);
if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) { if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) {
ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo, ret = decrypt_open(json_filename, maat_instance->decrypt_key,
(unsigned char **)&decrypted_buff, &decrypted_buff_sz, maat_instance->decrypt_algo,
err_str, err_str_sz); (unsigned char **)&decrypted_buff,
&decrypted_buff_sz,
err_str, err_str_sz);
if (ret < 0) { if (ret < 0) {
log_error(maat_instance->logger, MODULE_CONFIG_MONITOR, log_error(maat_instance->logger, MODULE_CONFIG_MONITOR,
"Decrypt Maat JSON file %s failed", json_filename); "Decrypt Maat JSON file %s failed", json_filename);
@@ -387,7 +390,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
} }
if (maat_instance->maat_json_is_gzipped) { if (maat_instance->maat_json_is_gzipped) {
ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz); ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff,
&uncompressed_buff_sz);
FREE(json_buff); FREE(json_buff);
if (ret < 0) { if (ret < 0) {
log_error(maat_instance->logger, MODULE_CONFIG_MONITOR, log_error(maat_instance->logger, MODULE_CONFIG_MONITOR,
@@ -410,7 +414,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
} }
ret = json2iris((const char*)json_buff, json_filename, NULL, ret = json2iris((const char*)json_buff, json_filename, NULL,
maat_instance->json_ctx.iris_file, sizeof(maat_instance->json_ctx.iris_file), maat_instance->json_ctx.iris_file,
sizeof(maat_instance->json_ctx.iris_file),
strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL, strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL,
strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL, strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL,
maat_instance->logger); maat_instance->logger);
@@ -422,17 +427,20 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
} }
if (!maat_instance->is_running) { if (!maat_instance->is_running) {
strncpy(maat_instance->json_ctx.json_file, json_filename, sizeof(maat_instance->json_ctx.json_file)); strncpy(maat_instance->json_ctx.json_file, json_filename,
sizeof(maat_instance->json_ctx.json_file));
} }
ret = stat(json_filename, &fstat_buf); ret = stat(json_filename, &fstat_buf);
maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim; maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim;
md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5); md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5);
log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, "JSON file %s md5: %s, generate index file %s OK", log_info(maat_instance->logger, MODULE_CONFIG_MONITOR,
maat_instance->json_ctx.json_file, "JSON file %s md5: %s, generate index file %s OK",
maat_instance->json_ctx.effective_json_md5, maat_instance->json_ctx.json_file,
maat_instance->json_ctx.iris_file); maat_instance->json_ctx.effective_json_md5,
maat_instance->json_ctx.iris_file);
maat_instance->input_mode = DATA_SOURCE_JSON_FILE; maat_instance->input_mode = DATA_SOURCE_JSON_FILE;
return 0; return 0;

70
src/maat_ex_data.cpp
View File

@@ -15,7 +15,6 @@
#include "uthash/utarray.h" #include "uthash/utarray.h"
#include "log/log.h" #include "log/log.h"
#include "rcu_hash.h" #include "rcu_hash.h"
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_ex_data.h" #include "maat_ex_data.h"
@@ -48,8 +47,9 @@ void cache_row_free(void *p)
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free}; UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn, struct ex_data_runtime *
struct log_handle *logger) ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn,
struct log_handle *logger)
{ {
struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1); struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1);
@@ -149,26 +149,31 @@ void ex_data_schema_free(struct ex_data_schema *ex_schema)
FREE(ex_schema); FREE(ex_schema);
} }
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema) void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt,
struct ex_data_schema *schema)
{ {
ex_data_rt->ex_schema = schema; ex_data_rt->ex_schema = schema;
} }
void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, struct ex_container_ctx *container_ctx) void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt,
struct ex_container_ctx *container_ctx)
{ {
rcu_hash_set_user_ctx(ex_data_rt->htable, container_ctx); rcu_hash_set_user_ctx(ex_data_rt->htable, container_ctx);
} }
struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt) struct ex_container_ctx *
ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt)
{ {
return (struct ex_container_ctx *)rcu_hash_get_user_ctx(ex_data_rt->htable); return (struct ex_container_ctx *)rcu_hash_get_user_ctx(ex_data_rt->htable);
} }
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len) void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row,
const char *key, size_t key_len)
{ {
void *ex_data = NULL; void *ex_data = NULL;
struct ex_data_schema *ex_schema = ex_data_rt->ex_schema; struct ex_data_schema *ex_schema = ex_data_rt->ex_schema;
ex_schema->new_func(ex_data_rt->table_id, key, row, &ex_data, ex_schema->argl, ex_schema->argp); ex_schema->new_func(ex_data_rt->table_id, key, row, &ex_data,
ex_schema->argl, ex_schema->argp);
return ex_data; return ex_data;
} }
@@ -194,11 +199,14 @@ void ex_data_container_free(void *ctx, void *data)
void *argp = container_ctx->ex_schema->argp; void *argp = container_ctx->ex_schema->argp;
struct ex_data_container *ex_container = (struct ex_data_container *)data; struct ex_data_container *ex_container = (struct ex_data_container *)data;
if (ex_container->ex_data != NULL && container_ctx->ex_schema->free_func != NULL) { if (ex_container->ex_data != NULL
container_ctx->ex_schema->free_func(container_ctx->table_id, &(ex_container->ex_data), argl, argp); && container_ctx->ex_schema->free_func != NULL) {
container_ctx->ex_schema->free_func(container_ctx->table_id,
&(ex_container->ex_data), argl, argp);
} }
if (ex_container->custom_data != NULL && container_ctx->custom_data_free != NULL) { if (ex_container->custom_data != NULL
&& container_ctx->custom_data_free != NULL) {
container_ctx->custom_data_free(ex_container->custom_data); container_ctx->custom_data_free(ex_container->custom_data);
} }
@@ -211,10 +219,12 @@ int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
{ {
struct ex_data_container *tmp_container = NULL; struct ex_data_container *tmp_container = NULL;
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (tmp_container != NULL) { if (tmp_container != NULL) {
log_error(ex_data_rt->logger, MODULE_EX_DATA, log_error(ex_data_rt->logger, MODULE_EX_DATA,
"ex_data_runtime add ex container error: already exist same key:%s", key); "ex_data_runtime add ex container error: already exist same key:%s",
key);
return -1; return -1;
} }
@@ -227,10 +237,12 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len) const char *key, size_t key_len)
{ {
struct ex_data_container *tmp_container = NULL; struct ex_data_container *tmp_container = NULL;
tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (NULL == tmp_container) { if (NULL == tmp_container) {
log_error(ex_data_rt->logger, MODULE_EX_DATA, log_error(ex_data_rt->logger, MODULE_EX_DATA,
"ex_data_runtime del ex container error: no such key:%s", key); "ex_data_runtime del ex container error: no such key:%s",
key);
return -1; return -1;
} }
@@ -239,22 +251,30 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
return 0; return 0;
} }
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{ {
struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); struct ex_data_container *ex_container = NULL;
ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (NULL == ex_container) { if (NULL == ex_container) {
return NULL; return NULL;
} }
void *dup_ex_data = NULL; void *dup_ex_data = NULL;
ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, &(ex_container->ex_data), ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data,
ex_data_rt->ex_schema->argl, ex_data_rt->ex_schema->argp); &(ex_container->ex_data),
ex_data_rt->ex_schema->argl,
ex_data_rt->ex_schema->argp);
return dup_ex_data; return dup_ex_data;
} }
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{ {
struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); struct ex_data_container *ex_container = NULL;
ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
key, key_len);
if (NULL == ex_container) { if (NULL == ex_container) {
return NULL; return NULL;
} }
@@ -267,12 +287,8 @@ size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt)
return rcu_hash_count(ex_data_rt->htable); return rcu_hash_count(ex_data_rt->htable);
} }
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container) size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container ***ex_container)
{ {
return rcu_hash_list_updating_data(ex_data_rt->htable, (void ***)ex_container); return rcu_hash_list_updating_data(ex_data_rt->htable, (void ***)ex_container);
} }
int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt)
{
return rcu_hash_updating_flag(ex_data_rt->htable);
}

215
src/maat_expr.cpp
View File

@@ -11,7 +11,6 @@
#include <string.h> #include <string.h>
#include <assert.h> #include <assert.h>
#include "utils.h"
#include "maat_expr.h" #include "maat_expr.h"
#include "adapter_hs.h" #include "adapter_hs.h"
#include "maat_utils.h" #include "maat_utils.h"
@@ -20,6 +19,8 @@
#include "rcu_hash.h" #include "rcu_hash.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_compile.h" #include "maat_compile.h"
#include "maat_group.h"
#include "alignment.h"
#include "maat_garbage_collection.h" #include "maat_garbage_collection.h"
#define MAX_DISTRICT_STR 128 #define MAX_DISTRICT_STR 128
@@ -36,6 +37,7 @@ struct expr_schema {
int is_hexbin_column; int is_hexbin_column;
enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ enum hs_scan_mode scan_mode; /* adapter_hs scan mode */
int table_id; //ugly int table_id; //ugly
struct table_manager *ref_tbl_mgr;
}; };
enum expr_type { enum expr_type {
@@ -56,7 +58,7 @@ enum match_method {
struct expr_item { struct expr_item {
int item_id; int item_id;
int group_id; int group_id;
char district[MAX_DISTRICT_STR]; int district_id;
char keywords[MAX_KEYWORDS_STR]; char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type; enum expr_type expr_type;
enum match_method match_method; enum match_method match_method;
@@ -80,9 +82,8 @@ struct expr_runtime {
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger; struct log_handle *logger;
// long long *scan_cnt; long long *scan_cnt;
// long long *hit_cnt; long long *hit_cnt;
// long long *not_grp_hit_cnt;
// long long *stream_num; // long long *stream_num;
}; };
@@ -139,11 +140,13 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
int db_hexbin = -1; int db_hexbin = -1;
int expr_type = -1; int expr_type = -1;
int match_method_type = -1; int match_method_type = -1;
enum table_type table_type = TABLE_TYPE_INVALID;
struct expr_item *expr_item = ALLOC(struct expr_item, 1); struct expr_item *expr_item = ALLOC(struct expr_item, 1);
int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len); int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no item_id", log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no item_id",
expr_schema->table_id, line); expr_schema->table_id, line);
goto error; goto error;
} }
@@ -151,32 +154,40 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no group_id", log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no group_id",
expr_schema->table_id, line); expr_schema->table_id, line);
goto error; goto error;
} }
expr_item->group_id = atoi(line + column_offset); expr_item->group_id = atoi(line + column_offset);
table_type = table_manager_get_table_type(expr_schema->tbl_mgr, expr_schema->table_id);
//TODO //TODO
#if 0 #if 1
if (table_item->table_type == TABLE_TYPE_EXPR_PLUS) { if (table_type == TABLE_TYPE_EXPR_PLUS) {
ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
return -1; goto error;
} }
if (column_len >= MAX_DISTRICT_STR) { if (column_len >= MAX_DISTRICT_STR) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"update error: expr table[%s]:item_id[%d] district length too long", "expr table(table_id:%d) line:%s district length too long",
table_name, expr_item->item_id); expr_schema->table_id, line);
return -1; goto error;
} }
memcpy(expr_item->district, (line + column_offset), column_len);
char district[MAX_DISTRICT_STR] = {0};
memcpy(district, (line + column_offset), column_len);
assert(strlen(district) > 0);
str_unescape(district);
expr_item->district_id = get_district_id()
} }
#endif #endif
ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no keywords", log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no keywords",
expr_schema->table_id, line); expr_schema->table_id, line);
goto error; goto error;
} }
@@ -191,7 +202,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no expr_type", log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no expr_type",
expr_schema->table_id, line); expr_schema->table_id, line);
goto error; goto error;
} }
@@ -201,7 +213,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no match_method", log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no match_method",
expr_schema->table_id, line); expr_schema->table_id, line);
goto error; goto error;
} }
@@ -211,7 +224,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no is_hexbin", log_error(logger, MODULE_EXPR,
"expr table(table_id:%d) line:%s has no is_hexbin",
expr_schema->table_id, line); expr_schema->table_id, line);
goto error; goto error;
} }
@@ -248,7 +262,8 @@ void expr_item_free(struct expr_item *expr_item)
FREE(expr_item); FREE(expr_item);
} }
void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{ {
int read_cnt = 0; int read_cnt = 0;
struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1); struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1);
@@ -267,15 +282,18 @@ void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *lo
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_EXPR, "table %s has no custom column", table_name); log_error(logger, MODULE_EXPR,
"table %s has no custom column", table_name);
goto error; goto error;
} }
custom_item = cJSON_GetObjectItem(item, "scan_mode"); custom_item = cJSON_GetObjectItem(item, "scan_mode");
if (custom_item != NULL && custom_item->type == cJSON_String) { if (custom_item != NULL && custom_item->type == cJSON_String) {
ret = maat_kv_read(scan_mode_map, custom_item->valuestring, (int*)&(expr_schema->scan_mode)); ret = maat_kv_read(scan_mode_map, custom_item->valuestring,
(int*)&(expr_schema->scan_mode));
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "scan_mode %s illegal", custom_item->valuestring); log_error(logger, MODULE_EXPR, "scan_mode %s illegal",
custom_item->valuestring);
goto error; goto error;
} }
read_cnt++; read_cnt++;
@@ -323,6 +341,8 @@ void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *lo
read_cnt++; read_cnt++;
} }
expr_schema->tbl_mgr = tbl_mgr;
if (read_cnt < 8) { if (read_cnt < 8) {
goto error; goto error;
} }
@@ -359,7 +379,8 @@ void expr_ex_data_free(void *user_ctx, void *data)
expr_rule_free(expr_rule); expr_rule_free(expr_rule);
} }
void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *expr_runtime_new(void *expr_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
if (NULL == expr_schema) { if (NULL == expr_schema) {
@@ -376,10 +397,8 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbag
expr_rt->ref_garbage_bin = garbage_bin; expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger; expr_rt->logger = logger;
// expr_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
// expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); expr_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
// expr_rt->not_grp_hit_cnt = alignment_int64_array_alloc(max_thread_num);
// expr_rt->stream_num = alignment_int64_array_alloc(max_thread_num);
return expr_rt; return expr_rt;
} }
@@ -411,11 +430,23 @@ void expr_runtime_free(void *expr_runtime)
HASH_DELETE(hh, expr_rt->item_hash, item); HASH_DELETE(hh, expr_rt->item_hash, item);
maat_item_free(item, expr_rt->item_user_data_free); maat_item_free(item, expr_rt->item_user_data_free);
} }
if (expr_rt->hit_cnt != NULL) {
alignment_int64_array_free(expr_rt->hit_cnt);
expr_rt->hit_cnt = NULL;
}
if (expr_rt->scan_cnt != NULL) {
alignment_int64_array_free(expr_rt->scan_cnt);
expr_rt->scan_cnt = NULL;
}
FREE(expr_rt); FREE(expr_rt);
} }
int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len, int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
and_expr_t *expr_rule, int is_valid, struct log_handle *logger) size_t key_len, and_expr_t *expr_rule,
int is_valid, struct log_handle *logger)
{ {
void *data = NULL; void *data = NULL;
@@ -424,7 +455,8 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_
data = rcu_hash_find(expr_rt->htable, key, key_len); data = rcu_hash_find(expr_rt->htable, key, key_len);
if (NULL == data) { if (NULL == data) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"the key of expr rule not exist, can't be deleted, expr_id:%d", expr_rule->expr_id); "the key of expr rule not exist, can't be deleted, expr_id:%d",
expr_rule->expr_id);
return -1; return -1;
} }
rcu_hash_del(expr_rt->htable, key, key_len); rcu_hash_del(expr_rt->htable, key, key_len);
@@ -433,7 +465,8 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_
data = rcu_hash_find(expr_rt->htable, key, key_len); data = rcu_hash_find(expr_rt->htable, key, key_len);
if (data != NULL) { if (data != NULL) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"the key of expr rule already exist, can't be added, expr_id:%d", expr_rule->expr_id); "the key of expr rule already exist, can't be added, expr_id:%d",
expr_rule->expr_id);
return -1; return -1;
} }
rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule); rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule);
@@ -461,7 +494,8 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type)
} }
#define MAAT_MAX_EXPR_ITEM_NUM 8 #define MAAT_MAX_EXPR_ITEM_NUM 8
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger) and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item,
struct log_handle *logger)
{ {
size_t i = 0; size_t i = 0;
size_t sub_expr_cnt = 0; size_t sub_expr_cnt = 0;
@@ -481,7 +515,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl
if (i >= MAAT_MAX_EXPR_ITEM_NUM) { if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"expr item_id:%d too many patterns", expr_item->item_id); "expr item_id:%d too many patterns",
expr_item->item_id);
return NULL; return NULL;
} }
@@ -515,7 +550,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl
return expr_rule; return expr_rule;
} }
int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column) int expr_runtime_update(void *expr_runtime, void *expr_schema,
const char *line, int valid_column)
{ {
if (NULL == expr_runtime || NULL == expr_schema) { if (NULL == expr_runtime || NULL == expr_schema) {
return -1; return -1;
@@ -553,7 +589,8 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line,
HASH_FIND_INT(expr_rt->item_hash, &item_id, item); HASH_FIND_INT(expr_rt->item_hash, &item_id, item);
if (item) { if (item) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
"expr runtime add item %d to item_hash failed, already exist", item_id); "expr runtime add item %d to item_hash failed, already exist",
item_id);
return -1; return -1;
} }
@@ -572,14 +609,16 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line,
expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger); expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger);
expr_item_free(expr_item); expr_item_free(expr_item);
if (NULL == expr_rule) { if (NULL == expr_rule) {
log_error(expr_rt->logger, MODULE_EXPR, "transform expr table(table_id:%d) item to expr_rule failed, item_id:%d", log_error(expr_rt->logger, MODULE_EXPR,
"transform expr table(table_id:%d) item to expr_rule failed, item_id:%d",
schema->table_id, item_id); schema->table_id, item_id);
return -1; return -1;
} }
} }
char *key = (char *)&item_id; char *key = (char *)&item_id;
ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, is_valid, expr_rt->logger); ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule,
is_valid, expr_rt->logger);
if (ret < 0) { if (ret < 0) {
if (expr_rule != NULL) { if (expr_rule != NULL) {
expr_rule_free(expr_rule); expr_rule_free(expr_rule);
@@ -622,20 +661,25 @@ int expr_runtime_commit(void *expr_runtime)
struct adapter_hs *old_adapter_hs = NULL; struct adapter_hs *old_adapter_hs = NULL;
log_info(expr_rt->logger, MODULE_EXPR, log_info(expr_rt->logger, MODULE_EXPR,
"committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt); "committing %zu expr rules for rebuilding adapter_hs engine",
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger); rule_cnt);
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode,
expr_rt->n_worker_thread,
rules, rule_cnt,
expr_rt->logger);
if (NULL == new_adapter_hs) { if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
"rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt); "rebuild adapter_hs engine failed when update %zu expr rules",
rule_cnt);
ret = -1; ret = -1;
} }
old_adapter_hs = expr_rt->hs; old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs; expr_rt->hs = new_adapter_hs;
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy); maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs,
(void (*)(void*))adapter_hs_destroy);
rcu_hash_commit(expr_rt->htable); rcu_hash_commit(expr_rt->htable);
expr_rt->rule_num = rcu_hash_count(expr_rt->htable);
rule_cnt = rcu_hash_updating_count(expr_rt->htable); rule_cnt = rcu_hash_updating_count(expr_rt->htable);
assert(rule_cnt == 0); assert(rule_cnt == 0);
@@ -645,71 +689,37 @@ int expr_runtime_commit(void *expr_runtime)
return ret; return ret;
} }
int expr_runtime_updating_flag(void *expr_runtime) int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{ {
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
return rcu_hash_updating_flag(expr_rt->htable);
}
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len,
int group_id_array[], size_t n_group_id_array, int virtual_table_id, struct maat_state *state)
{
if (NULL == expr_rt) {
return -1;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
size_t n_hit_item = 0; size_t n_hit_item = 0;
int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, hit_item_ids, &n_hit_item); int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, hit_item_ids, &n_hit_item);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
if (0 == n_hit_item) {
return 0;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(expr_rt->logger, MODULE_EXPR,
"hit expr item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
} }
struct maat_compile_state *compile_state = state->compile_mid; size_t group_hit_cnt = 0;
//tranform item_id to group_id ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, n_hit_item,
struct maat_item *item = NULL; group_ids, group_ids_size, &group_hit_cnt, state);
size_t n_group_id = 0; if (ret < 0) {
size_t i = 0; return -1;
for (i = 0; i < n_hit_item; i++) {
HASH_FIND_INT(expr_rt->item_hash, &(hit_item_ids[i]), item);
assert(item != NULL);
if (!item) {
// should not come here
continue;
}
if (n_group_id >= n_group_id_array) {
n_group_id = n_group_id_array;
//Prevent group_id_array out of bounds
} else {
group_id_array[n_group_id++] = item->group_id;
}
maat_compile_state_update_hit_path(compile_state, hit_item_ids[i], item->group_id, virtual_table_id, state->scan_cnt, i);
} }
// literal_id{group_id,vt_id} to clause_id return group_hit_cnt;
// STEP 1: get compile table runtime
int compile_table_id = -1;
if (state->compile_table_id == -1) {
compile_table_id = state->maat_instance->default_compile_table_id;
} else {
compile_table_id = state->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id);
enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, compile_table_id);
assert(table_type == TABLE_TYPE_COMPILE);
// STEP 2: get the specified compile table's hit clause_id array by literal_id
for (i = 0; i < n_group_id; i++) {
maat_compile_state_update_hit_clause(compile_state, compile_rt, group_id_array[i], virtual_table_id);
}
return n_group_id;
} }
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
@@ -722,14 +732,15 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
expr_rt->hs_stream = hs_stream; expr_rt->hs_stream = hs_stream;
} }
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data,
int result[], size_t *n_result) size_t data_len, int result[], size_t *n_result)
{ {
if (NULL == expr_rt) { if (NULL == expr_rt) {
return -1; return -1;
} }
return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, result, n_result); return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len,
result, n_result);
} }
void expr_runtime_stream_close(struct expr_runtime *expr_rt) void expr_runtime_stream_close(struct expr_runtime *expr_rt)
@@ -739,3 +750,13 @@ void expr_runtime_stream_close(struct expr_runtime *expr_rt)
expr_rt->hs_stream = NULL; expr_rt->hs_stream = NULL;
} }
} }
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id)
{
alignment_int64_array_add(expr_rt->hit_cnt, thread_id, 1);
}
long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread)
{
return alignment_int64_array_sum(expr_rt->hit_cnt, n_thread);
}

7
src/maat_garbage_collection.cpp
View File

@@ -8,13 +8,13 @@
*********************************************************************************************** ***********************************************************************************************
*/ */
#include "utils.h"
#include <time.h> #include <time.h>
#include <stdlib.h> #include <stdlib.h>
#include <sys/queue.h> #include <sys/queue.h>
#include <assert.h> #include <assert.h>
#include "maat_utils.h"
struct maat_garbage_bag { struct maat_garbage_bag {
time_t create_time; time_t create_time;
int timeout; int timeout;
@@ -60,7 +60,8 @@ size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin)
return bin->bag_cnt; return bin->bag_cnt;
} }
void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void (* func)(void *)) void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage,
void (* func)(void *))
{ {
struct maat_garbage_bag *bag = ALLOC(struct maat_garbage_bag, 1); struct maat_garbage_bag *bag = ALLOC(struct maat_garbage_bag, 1);

147
src/maat_group.cpp
View File

@@ -12,7 +12,6 @@
#include <pthread.h> #include <pthread.h>
#include "maat_group.h" #include "maat_group.h"
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "uthash/uthash.h" #include "uthash/uthash.h"
#include "igraph/igraph.h" #include "igraph/igraph.h"
@@ -22,23 +21,28 @@
struct group2group_item { struct group2group_item {
int group_id; int group_id;
int superior_group_id; int super_group_id;
}; };
struct group2group_schema { struct group2group_schema {
int group_id_column; int group_id_column;
int superior_group_id_column; int super_group_id_column;
int table_id;//ugly int table_id;//ugly
struct table_manager *tbl_mgr;
}; };
struct group2group_runtime { struct maat_group {
struct maat_group_topology *group_topo; igraph_integer_t vertex_id;
int group_id;
int ref_by_compile_cnt;
int ref_by_super_group_cnt;
int ref_by_sub_group_cnt;
uint32_t rule_num; size_t top_group_cnt;
uint32_t updating_rule_num; int *top_group_ids;
struct maat_garbage_bin *ref_garbage_bin; UT_hash_handle hh_group_id;
struct log_handle *logger; UT_hash_handle hh_vertex_id;
}; };
struct maat_group_topology { struct maat_group_topology {
@@ -53,7 +57,18 @@ struct maat_group_topology {
struct log_handle *logger; struct log_handle *logger;
}; };
void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) struct group2group_runtime {
struct maat_group_topology *group_topo;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
};
void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{ {
int read_cnt = 0; int read_cnt = 0;
struct group2group_schema *g2g_schema = ALLOC(struct group2group_schema, 1); struct group2group_schema *g2g_schema = ALLOC(struct group2group_schema, 1);
@@ -67,7 +82,8 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_GROUP, "table %s has no custom column", table_name); log_error(logger, MODULE_GROUP,
"table %s has no custom column", table_name);
goto error; goto error;
} }
@@ -77,12 +93,14 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "superior_group_id"); custom_item = cJSON_GetObjectItem(item, "super_group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2g_schema->superior_group_id_column = custom_item->valueint; g2g_schema->super_group_id_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
g2g_schema->tbl_mgr = tbl_mgr;
if (read_cnt < 3) { if (read_cnt < 3) {
goto error; goto error;
} }
@@ -117,7 +135,8 @@ struct maat_group_topology *maat_group_topology_new(struct log_handle *logger)
return group_topo; return group_topo;
} }
void *group2group_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *group2group_runtime_new(void *g2g_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
struct group2group_runtime *g2g_rt = ALLOC(struct group2group_runtime, 1); struct group2group_runtime *g2g_rt = ALLOC(struct group2group_runtime, 1);
@@ -167,6 +186,16 @@ void group2group_runtime_free(void *g2g_runtime)
FREE(g2g_rt); FREE(g2g_rt);
} }
void maat_group_ref_inc(struct maat_group *group)
{
group->ref_by_compile_cnt++;
}
void maat_group_ref_dec(struct maat_group *group)
{
group->ref_by_compile_cnt--;
}
struct group2group_item * struct group2group_item *
group2group_item_new(const char *line, struct group2group_schema *g2g_schema, group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
struct log_handle *logger) struct log_handle *logger)
@@ -184,14 +213,15 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
} }
g2g_item->group_id = atoi(line + column_offset); g2g_item->group_id = atoi(line + column_offset);
ret = get_column_pos(line, g2g_schema->superior_group_id_column, &column_offset, &column_len); ret = get_column_pos(line, g2g_schema->super_group_id_column,
&column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_GROUP, log_error(logger, MODULE_GROUP,
"group2group table(table_id:%d) line:%s has no superior_group_id", "group2group table(table_id:%d) line:%s has no super_group_id",
g2g_schema->table_id, line); g2g_schema->table_id, line);
goto error; goto error;
} }
g2g_item->superior_group_id = atoi(line + column_offset); g2g_item->super_group_id = atoi(line + column_offset);
return g2g_item; return g2g_item;
error: error:
@@ -251,13 +281,14 @@ void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *grou
struct maat_group_topology *group_topo = g2g_rt->group_topo; struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL); assert(group_topo != NULL);
assert(group->ref_by_compile_cnt == 0 && group->ref_by_superior_group_cnt == 0); assert(group->ref_by_compile_cnt == 0 && group->ref_by_super_group_cnt == 0);
igraph_vector_init(&v, 8); igraph_vector_init(&v, 8);
igraph_neighbors(&group_topo->group_graph, &v, group->vertex_id, IGRAPH_ALL); igraph_neighbors(&group_topo->group_graph, &v, group->vertex_id, IGRAPH_ALL);
if (igraph_vector_size(&v) > 0) { if (igraph_vector_size(&v) > 0) {
print_igraph_vector(&v, buff, sizeof(buff)); print_igraph_vector(&v, buff, sizeof(buff));
log_error(group_topo->logger, MODULE_GROUP, "Del group %d exception, still reached by %s.", log_error(group_topo->logger, MODULE_GROUP,
group->vertex_id, buff); "Del group %d exception, still reached by %s.",
group->vertex_id, buff);
assert(0); assert(0);
} }
igraph_vector_destroy(&v); igraph_vector_destroy(&v);
@@ -288,7 +319,8 @@ struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_i
return group; return group;
} }
int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int superior_group_id) int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id,
int super_group_id)
{ {
if (NULL == g2g_runtime) { if (NULL == g2g_runtime) {
return -1; return -1;
@@ -305,30 +337,34 @@ int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int
group = group2group_runtime_add_group(g2g_runtime, group_id); group = group2group_runtime_add_group(g2g_runtime, group_id);
} }
struct maat_group *superior_group = group2group_runtime_find_group(g2g_runtime, superior_group_id); struct maat_group *super_group = group2group_runtime_find_group(g2g_runtime,
if (NULL == superior_group) { super_group_id);
superior_group = group2group_runtime_add_group(g2g_runtime, superior_group_id); if (NULL == super_group) {
super_group = group2group_runtime_add_group(g2g_runtime, super_group_id);
} }
ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id,
super_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
//No duplicated edges between two groups. //No duplicated edges between two groups.
if (edge_id > 0) { if (edge_id > 0) {
log_error(g2g_rt->logger, MODULE_GROUP, log_error(g2g_rt->logger, MODULE_GROUP,
"Add group %d to group %d failed, relation already exisited.", "Add group %d to group %d failed, relation already exisited.",
group->group_id, superior_group->group_id); group->group_id, super_group->group_id);
ret = -1; ret = -1;
} else { } else {
igraph_add_edge(&group_topo->group_graph, group->vertex_id, superior_group->vertex_id); igraph_add_edge(&group_topo->group_graph, group->vertex_id,
group->ref_by_superior_group_cnt++; super_group->vertex_id);
superior_group->ref_by_subordinate_group_cnt++; group->ref_by_super_group_cnt++;
super_group->ref_by_sub_group_cnt++;
ret = 0; ret = 0;
} }
return ret; return ret;
} }
int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, int superior_group_id) int group2group_runtime_remove_group_from_group(void *g2g_runtime,
int group_id, int super_group_id)
{ {
if (NULL == g2g_runtime) { if (NULL == g2g_runtime) {
return -1; return -1;
@@ -341,15 +377,16 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id,
if (NULL == group) { if (NULL == group) {
log_error(g2g_rt->logger, MODULE_GROUP, log_error(g2g_rt->logger, MODULE_GROUP,
"Del group %d from group %d failed, group %d not exisited.", "Del group %d from group %d failed, group %d not exisited.",
group_id, superior_group_id, group_id); group_id, super_group_id, group_id);
return -1; return -1;
} }
struct maat_group *superior_group = group2group_runtime_find_group(g2g_runtime, superior_group_id); struct maat_group *super_group = group2group_runtime_find_group(g2g_runtime,
if (NULL == superior_group) { super_group_id);
if (NULL == super_group) {
log_error(g2g_rt->logger, MODULE_GROUP, log_error(g2g_rt->logger, MODULE_GROUP,
"Del group %d from group %d failed, superior group %d not exisited.", "Del group %d from group %d failed, superior group %d not exisited.",
group_id, superior_group_id, superior_group_id); group_id, super_group_id, super_group_id);
return -1; return -1;
} }
@@ -364,7 +401,8 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id,
//first edge, the fifth is the first vertex of the second edge and so on. The last element //first edge, the fifth is the first vertex of the second edge and so on. The last element
//of the argument list must be -1 to denote the end of the argument list. //of the argument list must be -1 to denote the end of the argument list.
//https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small
int ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1); int ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id,
super_group->vertex_id, -1);
assert(ret==IGRAPH_SUCCESS); assert(ret==IGRAPH_SUCCESS);
// ignore no such edge to abort(). // ignore no such edge to abort().
igraph_set_error_handler(igraph_error_handler_ignore); igraph_set_error_handler(igraph_error_handler_ignore);
@@ -377,8 +415,8 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id,
return -1; return -1;
} }
group->ref_by_superior_group_cnt--; group->ref_by_super_group_cnt--;
superior_group->ref_by_subordinate_group_cnt--; super_group->ref_by_sub_group_cnt--;
return 0; return 0;
} }
@@ -405,7 +443,7 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
} }
struct maat_group *group = NULL, *tmp = NULL; struct maat_group *group = NULL, *tmp = NULL;
struct maat_group *superior_group = NULL; struct maat_group *super_group = NULL;
int tmp_vid=0; int tmp_vid=0;
size_t top_group_cnt=0; size_t top_group_cnt=0;
int* temp_group_ids=NULL; int* temp_group_ids=NULL;
@@ -426,10 +464,11 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
HASH_ITER (hh_group_id, group_topo->hash_group_by_id, group, tmp) { HASH_ITER (hh_group_id, group_topo->hash_group_by_id, group, tmp) {
top_group_cnt = 0; top_group_cnt = 0;
temp_group_ids = NULL; temp_group_ids = NULL;
//Orphan, Not reference by any one, free it. //Orphan, Not reference by any one, free it.
if (0 == group->ref_by_compile_cnt if (0 == group->ref_by_compile_cnt
&& 0 == group->ref_by_superior_group_cnt && 0 == group->ref_by_super_group_cnt
&& 0 == group->ref_by_subordinate_group_cnt) { && 0 == group->ref_by_sub_group_cnt) {
FREE(group->top_group_ids); FREE(group->top_group_ids);
group2group_runtime_remove_group(g2g_runtime, group); group2group_runtime_remove_group(g2g_runtime, group);
@@ -437,8 +476,8 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
} }
//A group is need to build top groups when it has items and referenced by superior groups or compiles. //A group is need to build top groups when it has items and referenced by superior groups or compiles.
if (group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0) { if (group->ref_by_compile_cnt > 0 || group->ref_by_super_group_cnt > 0) {
if (0 == group->ref_by_superior_group_cnt) { if (0 == group->ref_by_super_group_cnt) {
//fast path, group is only referenced by compile rules. //fast path, group is only referenced by compile rules.
top_group_cnt = 1; top_group_cnt = 1;
temp_group_ids = ALLOC(int, top_group_cnt); temp_group_ids = ALLOC(int, top_group_cnt);
@@ -456,11 +495,12 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
break; break;
} }
HASH_FIND(hh_vertex_id, group_topo->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group); HASH_FIND(hh_vertex_id, group_topo->hash_group_by_vertex, &tmp_vid,
sizeof(tmp_vid), super_group);
//including itself //including itself
if (superior_group->ref_by_compile_cnt > 0) { if (super_group->ref_by_compile_cnt > 0) {
temp_group_ids[top_group_cnt] = superior_group->group_id; temp_group_ids[top_group_cnt] = super_group->group_id;
top_group_cnt++; top_group_cnt++;
} }
} }
@@ -479,10 +519,11 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
return 0; return 0;
} }
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
int valid_column) const char *line, int valid_column)
{ {
if (NULL == g2g_runtime || NULL == g2g_schema || NULL == line) { if (NULL == g2g_runtime || NULL == g2g_schema ||
NULL == line) {
return -1; return -1;
} }
@@ -501,10 +542,12 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *
if (0 == is_valid) { if (0 == is_valid) {
//delete //delete
ret = group2group_runtime_remove_group_from_group(g2g_runtime, g2g_item->group_id, g2g_item->superior_group_id); ret = group2group_runtime_remove_group_from_group(g2g_runtime, g2g_item->group_id,
g2g_item->super_group_id);
} else { } else {
//add //add
ret = group2group_runtime_add_group_to_group(g2g_runtime, g2g_item->group_id, g2g_item->superior_group_id); ret = group2group_runtime_add_group_to_group(g2g_runtime, g2g_item->group_id,
g2g_item->super_group_id);
} }
group2group_item_free(g2g_item); group2group_item_free(g2g_item);
@@ -520,8 +563,8 @@ int group2group_runtime_commit(void *g2g_runtime)
return group2group_runtime_build_top_groups(g2g_runtime); return group2group_runtime_build_top_groups(g2g_runtime);
} }
int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t n_group_ids, int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids,
int *top_group_ids) size_t n_group_ids, int *top_group_ids)
{ {
if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) { if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) {
return -1; return -1;

529
src/maat_ip.cpp
View File

@@ -11,16 +11,15 @@
#include <stdint.h> #include <stdint.h>
#include <assert.h> #include <assert.h>
#include "utils.h"
#include "log/log.h" #include "log/log.h"
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_ex_data.h" #include "maat_ex_data.h"
#include "IPMatcher.h" #include "IPMatcher.h"
#include "maat_ip.h" #include "maat_ip.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_compile.h" #include "maat_compile.h"
#include "alignment.h"
#include "maat_garbage_collection.h" #include "maat_garbage_collection.h"
#define MODULE_IP module_name_str("maat.ip") #define MODULE_IP module_name_str("maat.ip")
@@ -30,7 +29,7 @@ struct port_range {
uint16_t max_port; uint16_t max_port;
}; };
struct ip_plus_schema { struct ip_schema {
int item_id_column; int item_id_column;
int group_id_column; int group_id_column;
int addr_type_column; int addr_type_column;
@@ -40,15 +39,10 @@ struct ip_plus_schema {
int sport_format_column; int sport_format_column;
int sport1_column; int sport1_column;
int sport2_column; int sport2_column;
int daddr_format_column;
int dip1_column;
int dip2_column;
int dport_format_column;
int dport1_column;
int dport2_column;
int proto_column; int proto_column;
int direction_column; int direction_column;
int table_id; //ugly int table_id; //ugly
struct table_manager *tbl_mgr;
}; };
struct ipv4_item_rule { struct ipv4_item_rule {
@@ -69,7 +63,7 @@ struct ipv6_item_rule {
uint16_t direction; /* 方向，0表示双向，1表示单向 */ uint16_t direction; /* 方向，0表示双向，1表示单向 */
}; };
struct ip_plus_item { struct ip_item {
int item_id; int item_id;
int group_id; int group_id;
int addr_type; int addr_type;
@@ -79,7 +73,7 @@ struct ip_plus_item {
}; };
}; };
struct ip_plus_runtime { struct ip_runtime {
struct ip_matcher* ip_matcher; struct ip_matcher* ip_matcher;
struct ex_data_runtime* ex_data_rt; struct ex_data_runtime* ex_data_rt;
@@ -90,187 +84,174 @@ struct ip_plus_runtime {
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger; struct log_handle *logger;
long long *scan_cnt;
long long *hit_cnt;
}; };
void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{ {
size_t read_cnt = 0; size_t read_cnt = 0;
struct ip_plus_schema *ip_plus_schema = ALLOC(struct ip_plus_schema, 1); struct ip_schema *ip_schema = ALLOC(struct ip_schema, 1);
cJSON *custom_item = NULL; cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) { if (item != NULL && item->type == cJSON_Number) {
ip_plus_schema->table_id = item->valueint; ip_schema->table_id = item->valueint;
read_cnt++; read_cnt++;
} }
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) { if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_IP, "ip_plus table %s has no custom column", table_name); log_error(logger, MODULE_IP,
"ip table %s has no custom column", table_name);
goto error; goto error;
} }
custom_item = cJSON_GetObjectItem(item, "item_id"); custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->item_id_column = custom_item->valueint; ip_schema->item_id_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "group_id"); custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->group_id_column = custom_item->valueint; ip_schema->group_id_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "addr_type"); custom_item = cJSON_GetObjectItem(item, "addr_type");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->addr_type_column = custom_item->valueint; ip_schema->addr_type_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "saddr_format"); custom_item = cJSON_GetObjectItem(item, "saddr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->saddr_format_column = custom_item->valueint; ip_schema->saddr_format_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "sip1"); custom_item = cJSON_GetObjectItem(item, "sip1");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sip1_column = custom_item->valueint; ip_schema->sip1_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "sip2"); custom_item = cJSON_GetObjectItem(item, "sip2");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sip2_column = custom_item->valueint; ip_schema->sip2_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "sport_format"); custom_item = cJSON_GetObjectItem(item, "sport_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport_format_column = custom_item->valueint; ip_schema->sport_format_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "sport1"); custom_item = cJSON_GetObjectItem(item, "sport1");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport1_column = custom_item->valueint; ip_schema->sport1_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "sport2"); custom_item = cJSON_GetObjectItem(item, "sport2");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport2_column = custom_item->valueint; ip_schema->sport2_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "daddr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->daddr_format_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "dip1");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dip1_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "dip2");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dip2_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "dport_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport_format_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "dport1");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport1_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "dport2");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport2_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "proto"); custom_item = cJSON_GetObjectItem(item, "proto");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->proto_column = custom_item->valueint; ip_schema->proto_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(item, "direction"); custom_item = cJSON_GetObjectItem(item, "direction");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->direction_column = custom_item->valueint; ip_schema->direction_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
if (read_cnt < 18) { ip_schema->tbl_mgr = tbl_mgr;
if (read_cnt < 12) {
goto error; goto error;
} }
return ip_plus_schema; return ip_schema;
error: error:
FREE(ip_plus_schema); FREE(ip_schema);
return NULL; return NULL;
} }
void ip_plus_schema_free(void *ip_plus_schema) void ip_schema_free(void *ip_schema)
{ {
FREE(ip_plus_schema); FREE(ip_schema);
} }
void *ip_plus_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *ip_runtime_new(void *ip_schema, int max_thread_num,
struct log_handle *logger) struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{ {
if (NULL == ip_plus_schema) { if (NULL == ip_schema) {
return NULL; return NULL;
} }
struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema; struct ip_schema *schema = (struct ip_schema *)ip_schema;
struct ip_plus_runtime *ip_plus_rt = ALLOC(struct ip_plus_runtime, 1); struct ip_runtime *ip_rt = ALLOC(struct ip_runtime, 1);
ip_plus_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger); ip_rt->ex_data_rt = ex_data_runtime_new(schema->table_id,
ip_plus_rt->item_user_data_free = maat_item_inner_free; ex_data_container_free,
ip_plus_rt->ref_garbage_bin = garbage_bin; logger);
ip_plus_rt->logger = logger; ip_rt->item_user_data_free = maat_item_inner_free;
ip_rt->ref_garbage_bin = garbage_bin;
ip_rt->logger = logger;
return ip_plus_rt; ip_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
ip_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
return ip_rt;
} }
void ip_plus_runtime_free(void *ip_plus_runtime) void ip_runtime_free(void *ip_runtime)
{ {
if (NULL == ip_plus_runtime) { if (NULL == ip_runtime) {
return; return;
} }
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
if (ip_plus_rt->ip_matcher != NULL) { if (ip_rt->ip_matcher != NULL) {
ip_matcher_free(ip_plus_rt->ip_matcher); ip_matcher_free(ip_rt->ip_matcher);
} }
if (ip_plus_rt->ex_data_rt != NULL) { if (ip_rt->ex_data_rt != NULL) {
ex_data_runtime_free(ip_plus_rt->ex_data_rt); ex_data_runtime_free(ip_rt->ex_data_rt);
} }
struct maat_item *item = NULL, *tmp_item = NULL; struct maat_item *item = NULL, *tmp_item = NULL;
HASH_ITER(hh, ip_plus_rt->item_hash, item, tmp_item) { HASH_ITER(hh, ip_rt->item_hash, item, tmp_item) {
HASH_DELETE(hh, ip_plus_rt->item_hash, item); HASH_DELETE(hh, ip_rt->item_hash, item);
maat_item_free(item, ip_plus_rt->item_user_data_free); maat_item_free(item, ip_rt->item_user_data_free);
} }
FREE(ip_plus_rt); if (ip_rt->hit_cnt != NULL) {
alignment_int64_array_free(ip_rt->hit_cnt);
ip_rt->hit_cnt = NULL;
}
if (ip_rt->scan_cnt != NULL) {
alignment_int64_array_free(ip_rt->scan_cnt);
ip_rt->scan_cnt = NULL;
}
FREE(ip_rt);
} }
struct ip_plus_item *ip_plus_item_new(const char *line, struct ip_plus_schema *ip_plus_schema, struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema,
struct log_handle *logger) struct log_handle *logger)
{ {
size_t column_offset = 0; size_t column_offset = 0;
size_t column_len = 0; size_t column_len = 0;
@@ -282,198 +263,209 @@ struct ip_plus_item *ip_plus_item_new(const char *line, struct ip_plus_schema *i
uint16_t sport2 = 0; uint16_t sport2 = 0;
uint16_t protocol = 0; uint16_t protocol = 0;
uint16_t direction = 0; uint16_t direction = 0;
struct ip_plus_item *ip_plus_item = ALLOC(struct ip_plus_item, 1); struct ip_item *ip_item = ALLOC(struct ip_item, 1);
int ret = get_column_pos(line, ip_plus_schema->item_id_column, &column_offset, &column_len); int ret = get_column_pos(line, ip_schema->item_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no item_id", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line); "ip plus table(table_id:%d) line:%s has no item_id",
ip_schema->table_id, line);
goto error; goto error;
} }
ip_plus_item->item_id = atoi(line + column_offset); ip_item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, ip_plus_schema->group_id_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no group_id", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line); "ip plus table(table_id:%d) line:%s has no group_id",
ip_schema->table_id, line);
goto error; goto error;
} }
ip_plus_item->group_id = atoi(line + column_offset); ip_item->group_id = atoi(line + column_offset);
ret = get_column_pos(line, ip_plus_schema->addr_type_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->addr_type_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no addr_type", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line); "ip plus table(table_id:%d) line:%s has no addr_type",
ip_schema->table_id, line);
goto error; goto error;
} }
ip_plus_item->addr_type = atoi(line + column_offset); ip_item->addr_type = atoi(line + column_offset);
if (ip_plus_item->addr_type != 4 && ip_plus_item->addr_type != 6) { if (ip_item->addr_type != IPv4 && ip_item->addr_type != IPv6) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has invalid addr type:%d", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line, ip_plus_item->addr_type); "ip table(table_id:%d) line:%s has invalid addr type:%d",
ip_schema->table_id, line, ip_item->addr_type);
goto error; goto error;
} }
ret = get_column_pos(line, ip_plus_schema->saddr_format_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->saddr_format_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no saddr_format", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line); "ip table(table_id:%d) line:%s has no saddr_format",
ip_schema->table_id, line);
goto error; goto error;
} }
memcpy(saddr_format, (line + column_offset), column_len); memcpy(saddr_format, (line + column_offset), column_len);
if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) { if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) {
log_error(logger, MODULE_IP, log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid saddr_format, should be range/mask/CIDR", "ip table(table_id:%d) line:%s has invalid saddr_format, should be range/mask/CIDR",
ip_plus_schema->table_id, line); ip_schema->table_id, line);
goto error; goto error;
} }
ret = get_column_pos(line, ip_plus_schema->sip1_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->sip1_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sip1", log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no sip1",
ip_plus_schema->table_id, line); ip_schema->table_id, line);
goto error; goto error;
} }
memcpy(sip1_str, (line + column_offset), column_len); memcpy(sip1_str, (line + column_offset), column_len);
ret = get_column_pos(line, ip_plus_schema->sip2_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->sip2_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sip2", log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no sip2",
ip_plus_schema->table_id, line); ip_schema->table_id, line);
goto error; goto error;
} }
memcpy(sip2_str, (line + column_offset), column_len); memcpy(sip2_str, (line + column_offset), column_len);
ret = get_column_pos(line, ip_plus_schema->sport_format_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->sport_format_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport_format", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line); "ip table(table_id:%d) line:%s has no sport_format",
ip_schema->table_id, line);
goto error; goto error;
} }
memcpy(sport_format, (line + column_offset), column_len); memcpy(sport_format, (line + column_offset), column_len);
if (IP_FORMAT_UNKNOWN == ip_format_str2int(sport_format)) { if (IP_FORMAT_UNKNOWN == ip_format_str2int(sport_format)) {
log_error(logger, MODULE_IP, log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid sport_format, should be range/mask/CIDR", "ip table(table_id:%d) line:%s has invalid sport_format, should be range/mask/CIDR",
ip_plus_schema->table_id, line); ip_schema->table_id, line);
goto error; goto error;
} }
ret = get_column_pos(line, ip_plus_schema->sport1_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->sport1_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport1", log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no sport1",
ip_plus_schema->table_id, line); ip_schema->table_id, line);
goto error; goto error;
} }
sport1 = atoi(line + column_offset); sport1 = atoi(line + column_offset);
ret = get_column_pos(line, ip_plus_schema->sport2_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->sport2_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport2", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line); "ip table(table_id:%d) line:%s has no sport2",
ip_schema->table_id, line);
goto error; goto error;
} }
sport2 = atoi(line + column_offset); sport2 = atoi(line + column_offset);
if (4 == ip_plus_item->addr_type) { if (IPv4 == ip_item->addr_type) {
ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, ret = ip_format2range(ip_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str,
&ip_plus_item->ipv4.min_sip, &ip_plus_item->ipv4.max_sip); &ip_item->ipv4.min_sip, &ip_item->ipv4.max_sip);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s ip_format2range(ip4) failed", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line); "ip table(table_id:%d) line:%s ip_format2range(ip4) failed",
ip_schema->table_id, line);
goto error; goto error;
} }
if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) {
ip_plus_item->ipv4.min_sport = sport1 & sport2; ip_item->ipv4.min_sport = sport1 & sport2;
ip_plus_item->ipv4.max_sport = sport1 | ~sport2; ip_item->ipv4.max_sport = sport1 | ~sport2;
} else { } else {
ip_plus_item->ipv4.min_sport = sport1; ip_item->ipv4.min_sport = sport1;
ip_plus_item->ipv4.max_sport = sport2; ip_item->ipv4.max_sport = sport2;
} }
ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->proto_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no proto", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line); "ip table(table_id:%d) line:%s has no proto",
ip_schema->table_id, line);
goto error; goto error;
} }
ip_plus_item->ipv4.proto = atoi(line + column_offset); ip_item->ipv4.proto = atoi(line + column_offset);
protocol = ip_plus_item->ipv4.proto; protocol = ip_item->ipv4.proto;
ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->direction_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no direction", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line); "ip table(table_id:%d) line:%s has no direction",
ip_schema->table_id, line);
goto error; goto error;
} }
ip_plus_item->ipv4.direction = atoi(line + column_offset); ip_item->ipv4.direction = atoi(line + column_offset);
direction = ip_plus_item->ipv4.direction; direction = ip_item->ipv4.direction;
} else { } else {
//ipv6 //ipv6
ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, ret = ip_format2range(ip_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str,
ip_plus_item->ipv6.min_sip, ip_plus_item->ipv6.max_sip); ip_item->ipv6.min_sip, ip_item->ipv6.max_sip);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s ip_format2range(ip6) failed", log_error(logger, MODULE_IP,
ip_plus_schema->table_id, line); "ip table(table_id:%d) line:%s ip_format2range(ip6) failed",
ip_schema->table_id, line);
goto error; goto error;
} }
if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) {
ip_plus_item->ipv6.min_sport = sport1 & sport2; ip_item->ipv6.min_sport = sport1 & sport2;
ip_plus_item->ipv6.max_sport = sport1 | ~sport2; ip_item->ipv6.max_sport = sport1 | ~sport2;
} else { } else {
ip_plus_item->ipv6.min_sport = sport1; ip_item->ipv6.min_sport = sport1;
ip_plus_item->ipv6.max_sport = sport2; ip_item->ipv6.max_sport = sport2;
} }
ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->proto_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no proto", log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no proto",
ip_plus_schema->table_id, line); ip_schema->table_id, line);
goto error; goto error;
} }
ip_plus_item->ipv6.proto = atoi(line + column_offset); ip_item->ipv6.proto = atoi(line + column_offset);
protocol = ip_plus_item->ipv6.proto; protocol = ip_item->ipv6.proto;
ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->direction_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no direction", log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no direction",
ip_plus_schema->table_id, line); ip_schema->table_id, line);
goto error; goto error;
} }
ip_plus_item->ipv6.direction = atoi(line + column_offset); ip_item->ipv6.direction = atoi(line + column_offset);
direction = ip_plus_item->ipv6.direction; direction = ip_item->ipv6.direction;
} }
if (protocol > 65535 || protocol < 0) { if (protocol > 65535 || protocol < 0) {
log_error(logger, MODULE_IP, log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid proto:%d", "ip table(table_id:%d) line:%s has invalid proto:%d",
ip_plus_schema->table_id, line, protocol); ip_schema->table_id, line, protocol);
goto error; goto error;
} }
if (direction != 0 && direction != 1) { if (direction != 0 && direction != 1) {
log_error(logger, MODULE_IP, log_error(logger, MODULE_IP,
"ip_plus table(table_id:%d) line:%s has invalid direction:%d", "ip table(table_id:%d) line:%s has invalid direction:%d",
ip_plus_schema->table_id, line, direction); ip_schema->table_id, line, direction);
goto error; goto error;
} }
return ip_plus_item; return ip_item;
error: error:
FREE(ip_plus_item); FREE(ip_item);
return NULL; return NULL;
} }
void ip_plus_item_free(struct ip_plus_item *ip_plus_item) void ip_item_free(struct ip_item *ip_item)
{ {
FREE(ip_plus_item); FREE(ip_item);
} }
void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule) void ip_item_to_ip_rule(struct ip_item *item, struct ip_rule *rule)
{ {
struct port_range *sport_range = ALLOC(struct port_range, 1); struct port_range *sport_range = ALLOC(struct port_range, 1);
if (4 == item->addr_type) { if (IPv4 == item->addr_type) {
rule->type = IPv4; rule->type = IPv4;
sport_range->min_port = item->ipv4.min_sport; sport_range->min_port = item->ipv4.min_sport;
sport_range->max_port = item->ipv4.max_sport; sport_range->max_port = item->ipv4.max_sport;
@@ -483,20 +475,22 @@ void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule)
rule->type = IPv6; rule->type = IPv6;
sport_range->min_port = item->ipv6.min_sport; sport_range->min_port = item->ipv6.min_sport;
sport_range->max_port = item->ipv6.max_sport; sport_range->max_port = item->ipv6.max_sport;
memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, sizeof(item->ipv6.min_sip)); memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip,
memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, sizeof(item->ipv6.max_sip)); sizeof(item->ipv6.min_sip));
memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip,
sizeof(item->ipv6.max_sip));
} }
rule->rule_id = item->item_id; rule->rule_id = item->item_id;
rule->user_tag = sport_range; rule->user_tag = sport_range;
} }
struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt) struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt)
{ {
return ip_plus_rt->ex_data_rt; return ip_rt->ex_data_rt;
} }
int ip_plus_runtime_update_row(struct ip_plus_runtime *rt, char *key, size_t key_len, int ip_runtime_update_row(struct ip_runtime *rt, char *key, size_t key_len,
struct ip_plus_item *item, int is_valid) struct ip_item *item, int is_valid)
{ {
int ret = -1; int ret = -1;
struct ex_data_runtime *ex_data_rt = rt->ex_data_rt; struct ex_data_runtime *ex_data_rt = rt->ex_data_rt;
@@ -519,31 +513,25 @@ int ip_plus_runtime_update_row(struct ip_plus_runtime *rt, char *key, size_t key
return 0; return 0;
} }
int ip_plus_runtime_updating_flag(void *ip_plus_runtime) int ip_runtime_update(void *ip_runtime, void *ip_schema,
const char *line, int valid_column)
{ {
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; if (NULL == ip_runtime || NULL == ip_schema || NULL == line) {
return ex_data_runtime_updating_flag(ip_plus_rt->ex_data_rt);
}
int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line,
int valid_column)
{
if (NULL == ip_plus_runtime || NULL == ip_plus_schema || NULL == line) {
return -1; return -1;
} }
struct maat_item *item = NULL; struct maat_item *item = NULL;
struct ip_plus_item *ip_plus_item = NULL; struct ip_item *ip_item = NULL;
struct maat_item_inner *u_para = NULL; struct maat_item_inner *u_para = NULL;
struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema; struct ip_schema *schema = (struct ip_schema *)ip_schema;
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
int item_id = get_column_value(line, schema->item_id_column); int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
} else if (0 == is_valid) { } else if (0 == is_valid) {
//delete //delete
HASH_FIND_INT(ip_plus_rt->item_hash, &item_id, item); HASH_FIND_INT(ip_rt->item_hash, &item_id, item);
if (NULL == item) { if (NULL == item) {
return -1; return -1;
} }
@@ -555,56 +543,58 @@ int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const ch
return -1; return -1;
} }
HASH_DELETE(hh, ip_plus_rt->item_hash, item); HASH_DELETE(hh, ip_rt->item_hash, item);
maat_garbage_bagging(ip_plus_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free); maat_garbage_bagging(ip_rt->ref_garbage_bin, u_para,
(void (*)(void *))maat_item_inner_free);
} else { } else {
//add //add
HASH_FIND_INT(ip_plus_rt->item_hash, &item_id, item); HASH_FIND_INT(ip_rt->item_hash, &item_id, item);
if (item) { if (item) {
log_error(ip_plus_rt->logger, MODULE_IP, log_error(ip_rt->logger, MODULE_IP,
"ip_plus runtime add item %d to item_hash failed, already exist", item_id); "ip runtime add item %d to item_hash failed, already exist",
item_id);
return -1; return -1;
} }
ip_plus_item = ip_plus_item_new(line, schema, ip_plus_rt->logger); ip_item = ip_item_new(line, schema, ip_rt->logger);
if (NULL == ip_plus_item) { if (NULL == ip_item) {
return -1; return -1;
} }
u_para = maat_item_inner_new(ip_plus_item->group_id, item_id, 0); u_para = maat_item_inner_new(ip_item->group_id, item_id, 0);
item = maat_item_new(item_id, ip_plus_item->group_id, u_para); item = maat_item_new(item_id, ip_item->group_id, u_para);
HASH_ADD_INT(ip_plus_rt->item_hash, item_id, item); HASH_ADD_INT(ip_rt->item_hash, item_id, item);
} }
char *key = (char *)&item_id; char *key = (char *)&item_id;
int ret = ip_plus_runtime_update_row(ip_plus_rt, key, sizeof(int), ip_plus_item, is_valid); int ret = ip_runtime_update_row(ip_rt, key, sizeof(int), ip_item, is_valid);
if (ret < 0) { if (ret < 0) {
if (ip_plus_item != NULL) { if (ip_item != NULL) {
ip_plus_item_free(ip_plus_item); ip_item_free(ip_item);
ip_plus_item = NULL; ip_item = NULL;
} }
return -1; return -1;
} else { } else {
if (0 == is_valid) { if (0 == is_valid) {
ip_plus_rt->rule_num--; ip_rt->rule_num--;
} else { } else {
ip_plus_rt->rule_num++; ip_rt->rule_num++;
} }
} }
return 0; return 0;
} }
int ip_plus_runtime_commit(void *ip_plus_runtime) int ip_runtime_commit(void *ip_runtime)
{ {
if (NULL == ip_plus_runtime) { if (NULL == ip_runtime) {
return -1; return -1;
} }
int ret = 0; int ret = 0;
struct ex_data_container **ex_container = NULL; struct ex_data_container **ex_container = NULL;
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
struct ex_data_runtime *ex_data_rt = ip_plus_rt->ex_data_rt; struct ex_data_runtime *ex_data_rt = ip_rt->ex_data_rt;
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (0 == rule_cnt) { if (0 == rule_cnt) {
@@ -615,8 +605,8 @@ int ip_plus_runtime_commit(void *ip_plus_runtime)
struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt); struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) { for (size_t i = 0; i < rule_cnt; i++) {
struct ip_plus_item *item = (struct ip_plus_item *)ex_container[i]->custom_data; struct ip_item *item = (struct ip_item *)ex_container[i]->custom_data;
ip_plus_item_to_ip_rule(item, &rules[i]); ip_item_to_ip_rule(item, &rules[i]);
} }
struct ip_matcher *new_ip_matcher = NULL; struct ip_matcher *new_ip_matcher = NULL;
@@ -624,21 +614,23 @@ int ip_plus_runtime_commit(void *ip_plus_runtime)
size_t mem_used = 0; size_t mem_used = 0;
if (rule_cnt > 0) { if (rule_cnt > 0) {
log_info(ip_plus_rt->logger, MODULE_IP, log_info(ip_rt->logger, MODULE_IP,
"committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt); "committing %zu ip rules for rebuilding ip_matcher engine",
rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) { if (NULL == new_ip_matcher) {
log_error(ip_plus_rt->logger, MODULE_IP, log_error(ip_rt->logger, MODULE_IP,
"rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt); "rebuild ip_matcher engine failed when update %zu ip rules",
rule_cnt);
ret = -1; ret = -1;
} }
} }
old_ip_matcher = ip_plus_rt->ip_matcher; old_ip_matcher = ip_rt->ip_matcher;
ip_plus_rt->ip_matcher = new_ip_matcher; ip_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(ip_plus_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); maat_garbage_bagging(ip_rt->ref_garbage_bin, old_ip_matcher,
(void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ex_data_rt); ex_data_runtime_commit(ex_data_rt);
ip_plus_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt);
FREE(rules); FREE(rules);
FREE(ex_container); FREE(ex_container);
@@ -646,66 +638,57 @@ int ip_plus_runtime_commit(void *ip_plus_runtime)
return ret; return ret;
} }
int ip_plus_runtime_scan_ipv4(struct ip_plus_runtime *ip_plus_rt, int thread_id, uint32_t ip_addr, int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, int ip_type,
int *group_id_array, size_t n_group_id_array, int virtual_table_id, uint8_t *ip_addr, int *group_ids, size_t group_id_size,
struct maat_state *state) int vtable_id, struct maat_state *state)
{ {
if (NULL == ip_plus_rt) {
return -1;
}
int n_hit_item = 0; int n_hit_item = 0;
struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct ip_data ip; struct ip_data scan_data;
ip.type = IPv4; if (ip_type == IPv4) {
ip.ipv4 = ip_addr; scan_data.type = IPv4;
scan_data.ipv4 = *(uint32_t *)ip_addr;
} else {
scan_data.type = IPv6;
for (int i = 0; i < 4; i++) {
scan_data.ipv6[i] = *((uint32_t *)ip_addr + i);
}
}
n_hit_item = ip_matcher_match(ip_rt->ip_matcher, &scan_data, scan_results, sizeof(scan_result));
if (n_hit_item <= 0) {
return n_hit_item;
}
n_hit_item = ip_matcher_match(ip_plus_rt->ip_matcher, &ip, scan_results, MAX_SCANNER_HIT_ITEM_NUM);
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(ip_rt->logger, MODULE_IP,
"hit ip item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
} }
struct maat_compile_state *compile_state = state->compile_mid; int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
//tranform item_id to group_id for (int i = 0; i < n_hit_item; i++) {
struct maat_item *item = NULL; hit_item_ids[i] = scan_results[i].rule_id;
size_t n_group_id = 0;
int i = 0;
for (i = 0; i < n_hit_item; i++) {
HASH_FIND_INT(ip_plus_rt->item_hash, &(scan_results[i].rule_id), item);
assert(item != NULL);
if (!item) {
// should not come here
continue;
}
if (n_group_id >= n_group_id_array) {
n_group_id = n_group_id_array;
//Prevent group_id_array out of bounds
} else {
group_id_array[n_group_id++] = item->group_id;
}
// update hit path
maat_compile_state_update_hit_path(compile_state, scan_results[i].rule_id, item->group_id,
virtual_table_id, state->scan_cnt, i);
} }
// update hit clause: literal_id{group_id,vt_id} to clause_id size_t group_hit_cnt = 0;
int compile_table_id = -1; int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item,
if (state->compile_table_id == -1) { group_ids, group_id_size, &group_hit_cnt, state);
compile_table_id = state->maat_instance->default_compile_table_id; if (ret < 0) {
} else { return -1;
compile_table_id = state->compile_table_id;
} }
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id); return group_hit_cnt;
enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, compile_table_id); }
assert(table_type == TABLE_TYPE_COMPILE);
void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id)
for (size_t idx = 0; idx < n_group_id; idx++) { {
maat_compile_state_update_hit_clause(compile_state, compile_rt, group_id_array[idx], virtual_table_id); alignment_int64_array_add(ip_rt->hit_cnt, thread_id, 1);
} }
return n_group_id; long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread)
{
return alignment_int64_array_sum(ip_rt->hit_cnt, n_thread);
} }

129
src/maat_ip_plugin.cpp
View File

@@ -10,9 +10,8 @@
#include <assert.h> #include <assert.h>
#include "maat_ip_plugin.h"
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_ip_plugin.h"
#include "maat_ex_data.h" #include "maat_ex_data.h"
#include "IPMatcher.h" #include "IPMatcher.h"
#include "maat_rule.h" #include "maat_rule.h"
@@ -37,6 +36,10 @@ struct ip_plugin_schema {
int rule_tag_column; int rule_tag_column;
struct ex_data_schema *ex_schema; struct ex_data_schema *ex_schema;
int table_id; //ugly int table_id; //ugly
struct table_manager *tbl_mgr;
unsigned long long update_err_cnt;
unsigned long long unmatch_tag_cnt;
}; };
struct ip_plugin_runtime { struct ip_plugin_runtime {
@@ -50,7 +53,8 @@ struct ip_plugin_runtime {
struct log_handle *logger; struct log_handle *logger;
}; };
void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{ {
size_t read_cnt = 0; size_t read_cnt = 0;
struct ip_plugin_schema *ip_plugin_schema = ALLOC(struct ip_plugin_schema, 1); struct ip_plugin_schema *ip_plugin_schema = ALLOC(struct ip_plugin_schema, 1);
@@ -64,7 +68,8 @@ void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handl
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) { if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_IP_PLUGIN, "table %s has no custom column", table_name); log_error(logger, MODULE_IP_PLUGIN,
"table %s has no custom column", table_name);
goto error; goto error;
} }
@@ -92,6 +97,8 @@ void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handl
read_cnt++; read_cnt++;
} }
ip_plugin_schema->tbl_mgr = tbl_mgr;
if (read_cnt < 5) { if (read_cnt < 5) {
goto error; goto error;
} }
@@ -127,52 +134,102 @@ struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema
return schema->ex_schema; return schema->ex_schema;
} }
struct ip_plugin_item *ip_plugin_item_new(const char *line, struct ip_plugin_schema *ip_plugin_schema, int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line,
struct log_handle *logger) struct log_handle *logger)
{ {
size_t column_offset = 0; size_t column_offset = 0;
size_t column_len = 0; size_t column_len = 0;
struct ip_plugin_item *ip_plugin_item = ALLOC(struct ip_plugin_item, 1); size_t n_tag = table_manager_accept_tags_count(schema->tbl_mgr);
int ret = get_column_pos(line, ip_plugin_schema->item_id_column, &column_offset, &column_len); if (schema->rule_tag_column > 0 && n_tag > 0) {
int ret = get_column_pos(line, schema->rule_tag_column,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) has no rule_tag, line:%s",
schema->table_id, line);
schema->update_err_cnt++;
return TAG_MATCH_ERR;
}
if (column_len > 2) {
char *tag_str = ALLOC(char, column_len + 1);
memcpy(tag_str, (line + column_offset), column_len);
ret = table_manager_accept_tags_match(schema->tbl_mgr, tag_str);
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) has invalid tag format, line:%s",
schema->table_id, line);
schema->update_err_cnt++;
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
schema->unmatch_tag_cnt++;
return TAG_MATCH_UNMATCHED;
}
}
}
return TAG_MATCH_MATCHED;
}
struct ip_plugin_item *
ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
struct log_handle *logger)
{
int ret = ip_plugin_accept_tag_match(schema, line, logger);
if (ret == TAG_MATCH_UNMATCHED) {
return NULL;
}
size_t column_offset = 0;
size_t column_len = 0;
struct ip_plugin_item *ip_plugin_item = ALLOC(struct ip_plugin_item, 1);
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, "ip plugin table(table_id:%d) line:%s has no item_id", log_error(logger, MODULE_IP_PLUGIN,
ip_plugin_schema->table_id, line); "ip plugin table(table_id:%d) line:%s has no item_id",
schema->table_id, line);
goto error; goto error;
} }
ip_plugin_item->item_id = atoi(line + column_offset); ip_plugin_item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, ip_plugin_schema->ip_type_column, &column_offset, &column_len); ret = get_column_pos(line, schema->ip_type_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, "ip plugin table(table_id:%d) line:%s has no ip_type", log_error(logger, MODULE_IP_PLUGIN,
ip_plugin_schema->table_id, line); "ip plugin table(table_id:%d) line:%s has no ip_type",
schema->table_id, line);
goto error; goto error;
} }
ip_plugin_item->ip_type = atoi(line + column_offset); ip_plugin_item->ip_type = atoi(line + column_offset);
if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) { if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s ip_type[%d] invalid", "ip_plugin table(table_id:%d) line:%s ip_type[%d] invalid",
ip_plugin_schema->table_id, line, ip_plugin_item->ip_type); schema->table_id, line, ip_plugin_item->ip_type);
goto error; goto error;
} }
ret = get_column_pos(line, ip_plugin_schema->start_ip_column, &column_offset, &column_len); ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s has no start_ip", "ip_plugin table(table_id:%d) line:%s has no start_ip",
ip_plugin_schema->table_id, line); schema->table_id, line);
goto error; goto error;
} }
strncpy(ip_plugin_item->start_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->start_ip))); strncpy(ip_plugin_item->start_ip, line + column_offset,
MIN(column_len, sizeof(ip_plugin_item->start_ip)));
ret = get_column_pos(line, ip_plugin_schema->end_ip_column, &column_offset, &column_len); ret = get_column_pos(line, schema->end_ip_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s has no end_ip", "ip_plugin table(table_id:%d) line:%s has no end_ip",
ip_plugin_schema->table_id, line); schema->table_id, line);
goto error; goto error;
} }
strncpy(ip_plugin_item->end_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->end_ip))); strncpy(ip_plugin_item->end_ip, line + column_offset,
MIN(column_len, sizeof(ip_plugin_item->end_ip)));
return ip_plugin_item; return ip_plugin_item;
error: error:
@@ -245,7 +302,8 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_
return 0; return 0;
} }
void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
if (NULL == ip_plugin_schema) { if (NULL == ip_plugin_schema) {
@@ -255,7 +313,8 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct m
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1); struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1);
ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger); ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id,
ex_data_container_free, logger);
ip_plugin_rt->ref_garbage_bin = garbage_bin; ip_plugin_rt->ref_garbage_bin = garbage_bin;
ip_plugin_rt->logger = logger; ip_plugin_rt->logger = logger;
@@ -298,10 +357,11 @@ void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule
rule->user_tag = NULL; rule->user_tag = NULL;
} }
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line, int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
int valid_column) const char *line, int valid_column)
{ {
if (NULL == ip_plugin_runtime || NULL == ip_plugin_schema || NULL == line) { if (NULL == ip_plugin_runtime || NULL == ip_plugin_schema ||
NULL == line) {
return -1; return -1;
} }
@@ -323,7 +383,8 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, co
} }
char *key = (char *)&item_id; char *key = (char *)&item_id;
int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, line, key, sizeof(int), ip_plugin_item, is_valid); int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, line, key,
sizeof(int), ip_plugin_item, is_valid);
if (ret < 0) { if (ret < 0) {
if (ip_plugin_item != NULL) { if (ip_plugin_item != NULL) {
FREE(ip_plugin_item); FREE(ip_plugin_item);
@@ -370,20 +431,22 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime)
if (rule_cnt > 0) { if (rule_cnt > 0) {
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN, log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt); "committing %zu ip_plugin rules for rebuilding ip_matcher engine",
rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) { if (NULL == new_ip_matcher) {
log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt); "rebuild ip_matcher engine failed when update %zu ip_plugin rules",
rule_cnt);
ret = -1; ret = -1;
} }
} }
old_ip_matcher = ip_plugin_rt->ip_matcher; old_ip_matcher = ip_plugin_rt->ip_matcher;
ip_plugin_rt->ip_matcher = new_ip_matcher; ip_plugin_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher,
(void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ex_data_rt); ex_data_runtime_commit(ex_data_rt);
ip_plugin_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt);
FREE(rules); FREE(rules);
FREE(ex_container); FREE(ex_container);
@@ -391,12 +454,6 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime)
return ret; return ret;
} }
int ip_plugin_runtime_updating_flag(void *ip_plugin_runtime)
{
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt);
}
struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime) struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime)
{ {
if (NULL == ip_plugin_runtime) { if (NULL == ip_plugin_runtime) {

2
src/maat_kv.cpp
View File

@@ -12,7 +12,7 @@
#include <ctype.h> #include <ctype.h>
#include "uthash/uthash.h" #include "uthash/uthash.h"
#include "utils.h" #include "maat_utils.h"
#define MAAT_KV_MAX_KEY_LEN 512 #define MAAT_KV_MAX_KEY_LEN 512

117
src/maat_plugin.cpp
View File

@@ -12,11 +12,12 @@
#include "log/log.h" #include "log/log.h"
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_rule.h"
#include "maat_plugin.h" #include "maat_plugin.h"
#include "maat_ex_data.h" #include "maat_ex_data.h"
#include "maat_limits.h" #include "maat_limits.h"
#include "maat_table.h"
#define MODULE_PLUGIN module_name_str("maat.plugin") #define MODULE_PLUGIN module_name_str("maat.plugin")
@@ -49,6 +50,10 @@ struct plugin_schema {
struct plugin_callback_schema cb[MAX_PLUGIN_PER_TABLE]; struct plugin_callback_schema cb[MAX_PLUGIN_PER_TABLE];
struct ex_data_schema *ex_schema; struct ex_data_schema *ex_schema;
int table_id; //ugly int table_id; //ugly
struct table_manager *tbl_mgr;
unsigned long long update_err_cnt;
unsigned long long unmatch_tag_cnt;
}; };
static int read_integer_array(char *string, int *array, int size) static int read_integer_array(char *string, int *array, int size)
@@ -68,7 +73,8 @@ static int read_integer_array(char *string, int *array, int size)
return i; return i;
} }
void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{ {
size_t read_cnt = 0; size_t read_cnt = 0;
struct plugin_schema *plugin_schema = ALLOC(struct plugin_schema, 1); struct plugin_schema *plugin_schema = ALLOC(struct plugin_schema, 1);
@@ -82,7 +88,8 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_PLUGIN, "table %s has no custom column", table_name); log_error(logger, MODULE_PLUGIN,
"table %s has no custom column", table_name);
goto error; goto error;
} }
@@ -121,6 +128,8 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *
} }
} }
plugin_schema->tbl_mgr = tbl_mgr;
if (read_cnt < 5) { if (read_cnt < 5) {
goto error; goto error;
} }
@@ -160,8 +169,9 @@ int plugin_table_add_callback(void *plugin_schema, int table_id,
size_t idx = schema->cb_cnt; size_t idx = schema->cb_cnt;
if (idx == MAX_PLUGIN_PER_TABLE) { if (idx == MAX_PLUGIN_PER_TABLE) {
log_error(logger, MODULE_PLUGIN, "the plugin number of table_id: %d exceed maxium:%d", log_error(logger, MODULE_PLUGIN,
table_id, MAX_PLUGIN_PER_TABLE); "the plugin number of table_id: %d exceed maxium:%d",
table_id, MAX_PLUGIN_PER_TABLE);
return -1; return -1;
} }
@@ -192,7 +202,8 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema)
} }
} }
int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns) int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema,
int *foreign_columns)
{ {
if (NULL == plugin_schema) { if (NULL == plugin_schema) {
return -1; return -1;
@@ -221,7 +232,8 @@ int plugin_table_set_ex_data_schema(void *plugin_schema,
if (schema->ex_schema != NULL) { if (schema->ex_schema != NULL) {
assert(0); assert(0);
log_error(logger, MODULE_PLUGIN, log_error(logger, MODULE_PLUGIN,
"Error: %s, EX data schema already registed", __FUNCTION__); "Error: %s, EX data schema already registed",
__FUNCTION__);
return -1; return -1;
} }
schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp); schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp);
@@ -239,7 +251,8 @@ struct ex_data_schema *plugin_table_get_ex_data_schema(void *plugin_schema)
return schema->ex_schema; return schema->ex_schema;
} }
void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, void *plugin_runtime_new(void *plugin_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
if (NULL == plugin_schema) { if (NULL == plugin_schema) {
@@ -248,7 +261,8 @@ void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_ga
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
struct plugin_runtime *plugin_rt = ALLOC(struct plugin_runtime, 1); struct plugin_runtime *plugin_rt = ALLOC(struct plugin_runtime, 1);
plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger); plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id,
ex_data_container_free, logger);
plugin_rt->ref_garbage_bin = garbage_bin; plugin_rt->ref_garbage_bin = garbage_bin;
plugin_rt->logger = logger; plugin_rt->logger = logger;
@@ -270,18 +284,12 @@ void plugin_runtime_free(void *plugin_runtime)
FREE(plugin_rt); FREE(plugin_rt);
} }
int plugin_runtime_updating_flag(void *plugin_runtime) int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
{ struct plugin_schema *plugin_schema,
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; const char *row, char *key, size_t key_len,
return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt); int is_valid)
}
int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_schema *plugin_schema,
const char *row, char *key, size_t key_len, int is_valid)
{ {
int ret = -1; int ret = -1;
void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL);
struct ex_data_schema *ex_schema = plugin_schema->ex_schema; struct ex_data_schema *ex_schema = plugin_schema->ex_schema;
/* already set plugin_table_schema's ex_data_schema */ /* already set plugin_table_schema's ex_data_schema */
@@ -294,6 +302,8 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_sc
} }
} else { } else {
// add // add
void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL);
ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container); ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
@@ -313,33 +323,77 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_sc
ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row); ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row);
} }
plugin_rt->acc_line_num++;
return 0; return 0;
} }
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line, int plugin_accept_tag_match(struct plugin_schema *schema, const char *line,
int valid_column) struct log_handle *logger)
{ {
if (NULL == plugin_runtime || NULL == plugin_runtime) { size_t column_offset = 0;
size_t column_len = 0;
size_t n_tag = table_manager_accept_tags_count(schema->tbl_mgr);
if (schema->rule_tag_column > 0 && n_tag > 0) {
int ret = get_column_pos(line, schema->rule_tag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_PLUGIN,
"plugin table(table_id:%d) has no rule_tag, line:%s",
schema->table_id, line);
schema->update_err_cnt++;
return TAG_MATCH_ERR;
}
if (column_len > 2) {
char *tag_str = ALLOC(char, column_len + 1);
memcpy(tag_str, (line + column_offset), column_len);
ret = table_manager_accept_tags_match(schema->tbl_mgr, tag_str);
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_PLUGIN,
"plugin table(table_id:%d) has invalid tag format, line:%s",
schema->table_id, line);
schema->update_err_cnt++;
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
schema->unmatch_tag_cnt++;
return TAG_MATCH_UNMATCHED;
}
}
}
return TAG_MATCH_MATCHED;
}
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
const char *line, int valid_column)
{
if (NULL == plugin_runtime || NULL == plugin_schema) {
return -1; return -1;
} }
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
} }
char *key = (char *)&item_id;
int ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid);
if (ret < 0) {
} else {
int ret = plugin_accept_tag_match(schema, line, plugin_rt->logger);
if (ret == TAG_MATCH_UNMATCHED) {
return -1;
} }
int item_id = get_column_value(line, schema->item_id_column);
char *key = (char *)&item_id;
ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid);
if (ret < 0) {
schema->update_err_cnt++;
return -1;
}
plugin_rt->acc_line_num++;
return 0; return 0;
} }
@@ -351,7 +405,6 @@ int plugin_runtime_commit(void *plugin_runtime)
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
ex_data_runtime_commit(plugin_rt->ex_data_rt); ex_data_runtime_commit(plugin_rt->ex_data_rt);
//table_rt->rule_num = ex_data_runtime_ex_container_count(table_rt->plugin_rt.ex_data_rt);
return 0; return 0;
} }

323
src/maat_redis_monitor.cpp
View File

@@ -11,8 +11,9 @@
#include <string.h> #include <string.h>
#include <assert.h> #include <assert.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <stdio.h>
#include "utils.h" #include "maat/maat.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_command.h" #include "maat_command.h"
#include "maat_config_monitor.h" #include "maat_config_monitor.h"
@@ -39,28 +40,34 @@ const char *foreign_key_prefix = "__FILE_";
const char *mr_op_str[] = {"DEL", "ADD", "RENEW_TIMEOUT"}; const char *mr_op_str[] = {"DEL", "ADD", "RENEW_TIMEOUT"};
char *get_foreign_cont_filename(const char *table_name, int rule_id, const char *foreign_key, const char *dir) char *get_foreign_cont_filename(const char *table_name, int rule_id,
const char *foreign_key, const char *dir)
{ {
char buffer[512] = {0}; char buffer[512] = {0};
snprintf(buffer, sizeof(buffer),"%s/%s-%d-%s", dir, table_name, rule_id, foreign_key); snprintf(buffer, sizeof(buffer),"%s/%s-%d-%s", dir,
table_name, rule_id, foreign_key);
char *filename = ALLOC(char, strlen(buffer) + 1); char *filename = ALLOC(char, strlen(buffer) + 1);
memcpy(filename, buffer, strlen(buffer)); memcpy(filename, buffer, strlen(buffer));
return filename; return filename;
} }
void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_foreign, void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns,
const char *dir, struct log_handle *logger) int n_foreign, const char *dir, struct log_handle *logger)
{ {
int foreign_key_size = 0; int foreign_key_size = 0;
p_rule->f_keys = ALLOC(struct foreign_key, n_foreign); p_rule->f_keys = ALLOC(struct foreign_key, n_foreign);
for (int i = 0; i < n_foreign; i++) { for (int i = 0; i < n_foreign; i++) {
const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line, foreign_columns[i], &foreign_key_size); const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line,
foreign_columns[i],
&foreign_key_size);
if (NULL == p_foreign) { if (NULL == p_foreign) {
log_error(logger, MODULE_REDIS_MONITOR, "Get %s,%lu foreign keys failed: No %dth column", log_error(logger, MODULE_REDIS_MONITOR,
p_rule->table_name, p_rule->rule_id, foreign_columns[i]); "Get %s,%lu foreign keys failed: No %dth column",
p_rule->table_name, p_rule->rule_id,
foreign_columns[i]);
continue; continue;
} }
@@ -70,7 +77,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
} }
if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) { if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) {
log_error(logger, MODULE_REDIS_MONITOR, "Get %s,%lu foreign key failed: Invalid source prefix %s", log_error(logger, MODULE_REDIS_MONITOR,
"Get %s,%lu foreign key failed: Invalid source prefix %s",
p_rule->table_name, p_rule->rule_id, p_foreign); p_rule->table_name, p_rule->rule_id, p_foreign);
continue; continue;
} }
@@ -80,13 +88,17 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
p_foreign += strlen(foreign_source_prefix); p_foreign += strlen(foreign_source_prefix);
if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) { if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) {
log_info(logger, MODULE_REDIS_MONITOR, "%s, %lu foreign key prefix %s is not recommended", log_info(logger, MODULE_REDIS_MONITOR,
"%s, %lu foreign key prefix %s is not recommended",
p_rule->table_name, p_rule->rule_id, p_foreign); p_rule->table_name, p_rule->rule_id, p_foreign);
} }
p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size+1); p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size+1);
memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign, foreign_key_size); memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign, foreign_key_size);
p_rule->f_keys[p_rule->n_foreign].filename = get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[p_rule->n_foreign].key, dir); p_rule->f_keys[p_rule->n_foreign].filename = get_foreign_cont_filename(p_rule->table_name,
p_rule->rule_id,
p_rule->f_keys[p_rule->n_foreign].key,
dir);
p_rule->n_foreign++; p_rule->n_foreign++;
} }
@@ -95,7 +107,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
} }
} }
int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int rule_num, struct maat *maat_instance, const char *dir) int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, struct maat *maat_instance, const char *dir)
{ {
int rule_with_foreign_key = 0; int rule_with_foreign_key = 0;
@@ -112,7 +125,8 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in
} }
int foreign_columns[8]; int foreign_columns[8];
int n_foreign_column = plugin_table_get_foreign_column((struct plugin_schema *)schema, foreign_columns); int n_foreign_column = plugin_table_get_foreign_column((struct plugin_schema *)schema,
foreign_columns);
if (0 == n_foreign_column) { if (0 == n_foreign_column) {
continue; continue;
} }
@@ -175,9 +189,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
redisReply* reply = NULL; redisReply* reply = NULL;
for (i = 0; i < rule_num; i++) { for (i = 0; i < rule_num; i++) {
snprintf(redis_cmd, sizeof(redis_cmd), "GET %s:%s,%lu", mr_key_prefix[rule_list[i].op], snprintf(redis_cmd, sizeof(redis_cmd),
rule_list[i].table_name, "GET %s:%s,%lu", mr_key_prefix[rule_list[i].op],
rule_list[i].rule_id); rule_list[i].table_name,
rule_list[i].rule_id);
ret = redisAppendCommand(c, redis_cmd); ret = redisAppendCommand(c, redis_cmd);
assert(ret == REDIS_OK); assert(ret == REDIS_OK);
} }
@@ -185,9 +200,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
for (i = 0; i < rule_num; i++) { for (i = 0; i < rule_num; i++) {
ret = maat_cmd_wrap_redis_get_reply(c, &reply); ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) { if (ret == REDIS_ERR) {
log_error(logger, MODULE_REDIS_MONITOR, "Redis GET %s:%s,%lu failed, redis server error", log_error(logger, MODULE_REDIS_MONITOR,
mr_key_prefix[rule_list[i].op], "Redis GET %s:%s,%lu failed, redis server error",
rule_list[i].table_name, rule_list[i].rule_id); mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id);
error_happened = 1; error_happened = 1;
break; break;
} }
@@ -199,9 +215,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
retry_ids[failed_cnt] = i; retry_ids[failed_cnt] = i;
failed_cnt++; failed_cnt++;
} else { } else {
log_error(logger, MODULE_REDIS_MONITOR, "Redis GET %s:%s,%lu failed", log_error(logger, MODULE_REDIS_MONITOR,
mr_key_prefix[rule_list[i].op], "Redis GET %s:%s,%lu failed",
rule_list[i].table_name, rule_list[i].rule_id); mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id);
error_happened = 1; error_happened = 1;
} }
} }
@@ -218,9 +235,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
int idx = 0; int idx = 0;
for (i = 0; i < failed_cnt; i++) { for (i = 0; i < failed_cnt; i++) {
idx = retry_ids[i]; idx = retry_ids[i];
snprintf(redis_cmd, sizeof(redis_cmd), "GET %s:%s,%lu", mr_key_prefix[MAAT_OP_DEL], snprintf(redis_cmd, sizeof(redis_cmd),
rule_list[idx].table_name, "GET %s:%s,%lu", mr_key_prefix[MAAT_OP_DEL],
rule_list[idx].rule_id); rule_list[idx].table_name,
rule_list[idx].rule_id);
ret = redisAppendCommand(c, redis_cmd); ret = redisAppendCommand(c, redis_cmd);
} }
@@ -228,7 +246,9 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
idx = retry_ids[i]; idx = retry_ids[i];
ret = maat_cmd_wrap_redis_get_reply(c, &reply); ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) { if (ret == REDIS_ERR) {
log_error(logger, MODULE_REDIS_MONITOR, "redis command %s failed, redis server error", redis_cmd); log_error(logger, MODULE_REDIS_MONITOR,
"redis command %s failed, redis server error",
redis_cmd);
FREE(retry_ids); FREE(retry_ids);
return -1; return -1;
} }
@@ -237,12 +257,14 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
rule_list[idx].table_line = maat_strdup(reply->str); rule_list[idx].table_line = maat_strdup(reply->str);
} else if(reply->type==REDIS_REPLY_ERROR) { } else if(reply->type==REDIS_REPLY_ERROR) {
//Deal with Redis response: "Loading Redis is loading the database in memory" //Deal with Redis response: "Loading Redis is loading the database in memory"
log_error(logger, MODULE_REDIS_MONITOR, "redis command %s error, reply type=%d, error str=%s", log_error(logger, MODULE_REDIS_MONITOR,
redis_cmd, reply->type, reply->str); "redis command %s error, reply type=%d, error str=%s",
redis_cmd, reply->type, reply->str);
} else { } else {
//Handle type "nil" //Handle type "nil"
log_error(logger, MODULE_REDIS_MONITOR, "redis command %s failed, reply type=%d", log_error(logger, MODULE_REDIS_MONITOR,
redis_cmd, reply->type); "redis command %s failed, reply type=%d",
redis_cmd, reply->type);
} }
freeReplyObject(reply); freeReplyObject(reply);
@@ -254,8 +276,8 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
return 0; return 0;
} }
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list,
int print_process, struct log_handle *logger) int rule_num, int print_process, struct log_handle *logger)
{ {
int max_redis_batch = 4096; int max_redis_batch = 4096;
int success_cnt = 0; int success_cnt = 0;
@@ -286,14 +308,17 @@ int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int
} }
int get_inc_key_list(long long instance_version, long long target_version, int get_inc_key_list(long long instance_version, long long target_version,
redisContext *c, struct serial_rule **list, struct log_handle *logger) redisContext *c, struct serial_rule **list,
struct log_handle *logger)
{ {
//Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version. //Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version.
//The elements are considered to be ordered from low to high scores(instance_version). //The elements are considered to be ordered from low to high scores(instance_version).
redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld", mr_status_sset, redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld",
instance_version,target_version); mr_status_sset, instance_version,
target_version);
if (NULL == reply) { if (NULL == reply) {
log_error(logger, MODULE_REDIS_MONITOR, "GET %s failed with a NULL reply, error: %s", log_error(logger, MODULE_REDIS_MONITOR,
"GET %s failed with a NULL reply, error: %s",
mr_status_sset, c->errstr); mr_status_sset, c->errstr);
return -1; return -1;
} }
@@ -307,10 +332,14 @@ int get_inc_key_list(long long instance_version, long long target_version,
return 0; return 0;
} }
redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s", mr_status_sset, reply->element[0]->str); redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s",
mr_status_sset,
reply->element[0]->str);
if (tmp_reply->type != REDIS_REPLY_STRING) { if (tmp_reply->type != REDIS_REPLY_STRING) {
log_error(logger, MODULE_REDIS_MONITOR, "ZSCORE %s %s failed Version: %lld->%lld", log_error(logger, MODULE_REDIS_MONITOR,
mr_status_sset, reply->element[0]->str, instance_version, target_version); "ZSCORE %s %s failed Version: %lld->%lld",
mr_status_sset, reply->element[0]->str,
instance_version, target_version);
freeReplyObject(tmp_reply); freeReplyObject(tmp_reply);
tmp_reply = NULL; tmp_reply = NULL;
freeReplyObject(reply); freeReplyObject(reply);
@@ -327,7 +356,8 @@ int get_inc_key_list(long long instance_version, long long target_version,
} }
if (nearest_rule_version != instance_version + 1) { if (nearest_rule_version != instance_version + 1) {
log_info(logger, MODULE_REDIS_MONITOR, "Noncontinuous VERSION Redis: %lld MAAT: %lld", log_info(logger, MODULE_REDIS_MONITOR,
"Noncontinuous VERSION Redis: %lld MAAT: %lld",
nearest_rule_version, instance_version); nearest_rule_version, instance_version);
} }
@@ -338,10 +368,11 @@ int get_inc_key_list(long long instance_version, long long target_version,
for (i = 0, j = 0; i < (int)reply->elements; i++) { for (i = 0, j = 0; i < (int)reply->elements; i++) {
assert(reply->element[i]->type == REDIS_REPLY_STRING); assert(reply->element[i]->type == REDIS_REPLY_STRING);
int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu", op_str, int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu",
s_rule[j].table_name, &(s_rule[j].rule_id)); op_str, s_rule[j].table_name, &(s_rule[j].rule_id));
if (ret != 3 || s_rule[i].rule_id < 0) { if (ret != 3 || s_rule[i].rule_id < 0) {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key: %s", reply->element[i]->str); log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key: %s", reply->element[i]->str);
continue; continue;
} }
@@ -350,7 +381,8 @@ int get_inc_key_list(long long instance_version, long long target_version,
} else if(strncmp(op_str, "DEL", strlen("DEL")) == 0) { } else if(strncmp(op_str, "DEL", strlen("DEL")) == 0) {
s_rule[j].op = MAAT_OP_DEL; s_rule[j].op = MAAT_OP_DEL;
} else { } else {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key: %s", reply->element[i]->str); log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key: %s", reply->element[i]->str);
continue; continue;
} }
j++; j++;
@@ -398,7 +430,8 @@ struct serial_rule *serial_rule_clone(const struct serial_rule *s_rule)
new_rule->f_keys[j].key = ALLOC(char, s_rule->f_keys[j].key_len); new_rule->f_keys[j].key = ALLOC(char, s_rule->f_keys[j].key_len);
memcpy(new_rule->f_keys[j].key, s_rule->f_keys[j].key, s_rule->f_keys[j].key_len); memcpy(new_rule->f_keys[j].key, s_rule->f_keys[j].key, s_rule->f_keys[j].key_len);
new_rule->f_keys[j].filename = ALLOC(char, strlen(s_rule->f_keys[j].filename)); new_rule->f_keys[j].filename = ALLOC(char, strlen(s_rule->f_keys[j].filename));
memcpy(new_rule->f_keys[j].filename, s_rule->f_keys[j].filename, strlen(s_rule->f_keys[j].filename)); memcpy(new_rule->f_keys[j].filename, s_rule->f_keys[j].filename,
strlen(s_rule->f_keys[j].filename));
} }
return new_rule; return new_rule;
@@ -466,10 +499,10 @@ int recovery_history_version(const struct serial_rule *current, int current_num,
return ret; return ret;
} }
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version,
long long *new_version, struct table_manager *tbl_mgr, long long desired_version, long long *new_version,
struct serial_rule **list, int *update_type, int cumulative_off, struct table_manager *tbl_mgr, struct serial_rule **list,
struct log_handle *logger) int *update_type, int cumulative_off, struct log_handle *logger)
{ {
int rule_num = 0; int rule_num = 0;
long long target_version = 0; long long target_version = 0;
@@ -478,20 +511,23 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
redisReply *reply = (redisReply *)redisCommand(c, "GET MAAT_VERSION"); redisReply *reply = (redisReply *)redisCommand(c, "GET MAAT_VERSION");
if (reply != NULL) { if (reply != NULL) {
if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) { if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) {
log_error(logger, MODULE_REDIS_MONITOR, "GET MAAT_VERSION failed, maybe Redis is busy"); log_error(logger, MODULE_REDIS_MONITOR,
"GET MAAT_VERSION failed, maybe Redis is busy");
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
return -1; return -1;
} }
} else { } else {
log_error(logger, MODULE_REDIS_MONITOR, "GET MAAT_VERSION failed with NULL reply, error: %s", c->errstr); log_error(logger, MODULE_REDIS_MONITOR,
"GET MAAT_VERSION failed with NULL reply, error: %s", c->errstr);
return -1; return -1;
} }
long long redis_version = maat_cmd_read_redis_integer(reply); long long redis_version = maat_cmd_read_redis_integer(reply);
if (redis_version < 0) { if (redis_version < 0) {
if (reply->type == REDIS_REPLY_ERROR) { if (reply->type == REDIS_REPLY_ERROR) {
log_error(logger, MODULE_REDIS_MONITOR, "Redis Communication error: %s", reply->str); log_error(logger, MODULE_REDIS_MONITOR,
"Redis Communication error: %s", reply->str);
} }
return -1; return -1;
} }
@@ -508,7 +544,8 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
} }
if (redis_version < instance_version) { if (redis_version < instance_version) {
log_error(logger, MODULE_REDIS_MONITOR, "VERSION roll back MAAT: %lld -> Redis: %lld", log_error(logger, MODULE_REDIS_MONITOR,
"VERSION roll back MAAT: %lld -> Redis: %lld",
instance_version, redis_version); instance_version, redis_version);
goto FULL_UPDATE; goto FULL_UPDATE;
} }
@@ -521,7 +558,8 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
do { do {
target_version++; target_version++;
rule_num = get_inc_key_list(instance_version, target_version, c, &s_rule_array, logger); rule_num = get_inc_key_list(instance_version, target_version,
c, &s_rule_array, logger);
if (rule_num > 0) { if (rule_num > 0) {
break; break;
} else if (rule_num < 0) { } else if (rule_num < 0) {
@@ -533,21 +571,25 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
} while (0 == rule_num && target_version <= redis_version && 1 == cumulative_off); } while (0 == rule_num && target_version <= redis_version && 1 == cumulative_off);
if (0 == rule_num) { if (0 == rule_num) {
log_info(logger, MODULE_REDIS_MONITOR, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s", log_info(logger, MODULE_REDIS_MONITOR,
mr_status_sset, instance_version, target_version-1, cumulative_off == 1 ? "OFF" : "ON"); "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s",
mr_status_sset, instance_version, target_version-1,
cumulative_off == 1 ? "OFF" : "ON");
return 0; return 0;
} }
log_info(logger, MODULE_REDIS_MONITOR, "Inc Update from instance_version %lld to %lld (%d entries)", log_info(logger, MODULE_REDIS_MONITOR,
"Inc Update from instance_version %lld to %lld (%d entries)",
instance_version, target_version, rule_num); instance_version, target_version, rule_num);
*list = s_rule_array; *list = s_rule_array;
*update_type = CM_UPDATE_TYPE_INC; *update_type = MAAT_UPDATE_TYPE_INC;
*new_version = target_version; *new_version = target_version;
return rule_num; return rule_num;
FULL_UPDATE: FULL_UPDATE:
log_info(logger, MODULE_REDIS_MONITOR, "Initiate full update from instance_version %lld to %lld", log_info(logger, MODULE_REDIS_MONITOR,
"Initiate full update from instance_version %lld to %lld",
instance_version, desired_version == 0 ? redis_version : desired_version); instance_version, desired_version == 0 ? redis_version : desired_version);
size_t append_cmd_cnt = 0; size_t append_cmd_cnt = 0;
int ret = redisAppendCommand(c, "MULTI"); int ret = redisAppendCommand(c, "MULTI");
@@ -569,12 +611,14 @@ FULL_UPDATE:
reply = maat_cmd_wrap_redis_command(c, "EXEC"); reply = maat_cmd_wrap_redis_command(c, "EXEC");
if (NULL == reply) { if (NULL == reply) {
log_error(logger, MODULE_REDIS_MONITOR, "Redis Communication error: %s", c->errstr); log_error(logger, MODULE_REDIS_MONITOR,
"Redis Communication error: %s", c->errstr);
return -1; return -1;
} }
if (reply->type != REDIS_REPLY_ARRAY) { if (reply->type != REDIS_REPLY_ARRAY) {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key List type %d", reply->type); log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key List type %d", reply->type);
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
return -1; return -1;
@@ -583,7 +627,8 @@ FULL_UPDATE:
*new_version = maat_cmd_read_redis_integer(reply->element[0]); *new_version = maat_cmd_read_redis_integer(reply->element[0]);
redisReply *sub_reply = reply->element[1]; redisReply *sub_reply = reply->element[1];
if (sub_reply->type != REDIS_REPLY_ARRAY) { if (sub_reply->type != REDIS_REPLY_ARRAY) {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key List type %d", sub_reply->type); log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key List type %d", sub_reply->type);
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
return -1; return -1;
@@ -593,17 +638,21 @@ FULL_UPDATE:
s_rule_array = ALLOC(struct serial_rule, sub_reply->elements); s_rule_array = ALLOC(struct serial_rule, sub_reply->elements);
for (i = 0, full_idx = 0; i < sub_reply->elements; i++) { for (i = 0, full_idx = 0; i < sub_reply->elements; i++) {
if (sub_reply->element[i]->type != REDIS_REPLY_STRING) { if (sub_reply->element[i]->type != REDIS_REPLY_STRING) {
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key Type: %d", sub_reply->element[i]->type); log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key Type: %d", sub_reply->element[i]->type);
continue; continue;
} }
ret = sscanf(sub_reply->element[i]->str, "%*[^:]:%[^,],%ld", ret = sscanf(sub_reply->element[i]->str, "%*[^:]:%[^,],%ld",
s_rule_array[full_idx].table_name, s_rule_array[full_idx].table_name,
&(s_rule_array[full_idx].rule_id)); &(s_rule_array[full_idx].rule_id));
s_rule_array[full_idx].op = MAAT_OP_ADD; s_rule_array[full_idx].op = MAAT_OP_ADD;
if (ret != 2 || s_rule_array[full_idx].rule_id < 0 || strlen(s_rule_array[full_idx].table_name) == 0) { if (ret != 2 || s_rule_array[full_idx].rule_id < 0 ||
log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key Format: %s", sub_reply->element[i]->str); strlen(s_rule_array[full_idx].table_name) == 0) {
log_error(logger, MODULE_REDIS_MONITOR,
"Invalid Redis Key Format: %s",
sub_reply->element[i]->str);
continue; continue;
} }
@@ -623,22 +672,27 @@ FULL_UPDATE:
if (desired_version != 0) { if (desired_version != 0) {
struct serial_rule *changed_rule_array = NULL; struct serial_rule *changed_rule_array = NULL;
int changed_rule_num = get_inc_key_list(desired_version, redis_version, c, &changed_rule_array, logger); int changed_rule_num = get_inc_key_list(desired_version, redis_version,
c, &changed_rule_array, logger);
if (changed_rule_num < 0) { if (changed_rule_num < 0) {
log_error(logger, MODULE_REDIS_MONITOR, "Recover history version %lld faild where as redis version is %lld", log_error(logger, MODULE_REDIS_MONITOR,
"Recover history version %lld faild where as redis version is %lld",
desired_version, redis_version); desired_version, redis_version);
} else if(0 == changed_rule_num) { } else if(0 == changed_rule_num) {
log_error(logger, MODULE_REDIS_MONITOR, "Nothing to recover from history version %lld to redis version is %lld", log_error(logger, MODULE_REDIS_MONITOR,
"Nothing to recover from history version %lld to redis version is %lld",
desired_version, redis_version); desired_version, redis_version);
} else { } else {
struct serial_rule *history_rule_array = NULL; struct serial_rule *history_rule_array = NULL;
ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array); ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array,
changed_rule_num, &history_rule_array);
if (ret > 0) { if (ret > 0) {
FREE(s_rule_array); FREE(s_rule_array);
s_rule_array = history_rule_array; s_rule_array = history_rule_array;
rule_num = ret; rule_num = ret;
*new_version = desired_version; *new_version = desired_version;
log_info(logger, MODULE_REDIS_MONITOR, "Successfully recovered from history version %lld to redis version is %lld", log_info(logger, MODULE_REDIS_MONITOR,
"Successfully recovered from history version %lld to redis version is %lld",
desired_version, redis_version); desired_version, redis_version);
} }
} }
@@ -646,14 +700,15 @@ FULL_UPDATE:
} }
*list = s_rule_array; *list = s_rule_array;
*update_type = CM_UPDATE_TYPE_FULL; *update_type = MAAT_UPDATE_TYPE_FULL;
log_info(logger, MODULE_REDIS_MONITOR, "Full update %d keys of version %lld", rule_num, *new_version); log_info(logger, MODULE_REDIS_MONITOR,
"Full update %d keys of version %lld", rule_num, *new_version);
return rule_num ; return rule_num ;
} }
void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn, void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
struct log_handle *logger) int rule_num, int print_fn, struct log_handle *logger)
{ {
int i = 0; int i = 0;
int j = 0; int j = 0;
@@ -676,8 +731,9 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
ret = remove(rule_list[i].f_keys[j].filename); ret = remove(rule_list[i].f_keys[j].filename);
if (ret == -1) { if (ret == -1) {
log_error(logger, MODULE_REDIS_MONITOR, "Foreign content file %s remove failed", log_error(logger, MODULE_REDIS_MONITOR,
rule_list[i].f_keys[j].filename); "Foreign content file %s remove failed",
rule_list[i].f_keys[j].filename);
} }
} }
} else { } else {
@@ -734,7 +790,8 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
fclose(fp); fclose(fp);
fp = NULL; fp = NULL;
if (1 == print_fn) { if (1 == print_fn) {
printf("Written foreign content %s\n", s_rule->f_keys[track[i].foreign_idx].filename); printf("Written foreign content %s\n",
s_rule->f_keys[track[i].foreign_idx].filename);
} }
} }
} }
@@ -747,8 +804,8 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
return; return;
} }
void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn, void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
struct log_handle *logger) int rule_num, int print_fn, struct log_handle *logger)
{ {
int max_redis_batch = 4096; int max_redis_batch = 4096;
int success_cnt = 0; int success_cnt = 0;
@@ -797,12 +854,14 @@ void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule)
pos_rewrite_line += strlen(s_rule->f_keys[i].filename); pos_rewrite_line += strlen(s_rule->f_keys[i].filename);
} }
strncat(pos_rewrite_line, pos_origin_line, strlen(s_rule->table_line) - (pos_origin_line - s_rule->table_line)); strncat(pos_rewrite_line, pos_origin_line,
strlen(s_rule->table_line) - (pos_origin_line - s_rule->table_line));
FREE(s_rule->table_line); FREE(s_rule->table_line);
s_rule->table_line = rewrite_line; s_rule->table_line = rewrite_line;
} }
void expected_reply_add(struct expected_reply* expected, int s_rule_seq, int type, long long integer) void expected_reply_add(struct expected_reply* expected, int s_rule_seq,
int type, long long integer)
{ {
int i = expected->possible_reply_num; int i = expected->possible_reply_num;
assert(i < POSSIBLE_REDIS_REPLY_SIZE); assert(i < POSSIBLE_REDIS_REPLY_SIZE);
@@ -816,7 +875,8 @@ int redlock_try_lock(redisContext *c, const char *lock_name, long long expire)
{ {
int ret = 0; int ret = 0;
redisReply *reply = maat_cmd_wrap_redis_command(c, "SET %s locked NX PX %lld", lock_name, expire); redisReply *reply = maat_cmd_wrap_redis_command(c, "SET %s locked NX PX %lld",
lock_name, expire);
if (reply->type == REDIS_REPLY_NIL) { if (reply->type == REDIS_REPLY_NIL) {
ret = 0; ret = 0;
} else { } else {
@@ -829,8 +889,9 @@ int redlock_try_lock(redisContext *c, const char *lock_name, long long expire)
return ret; return ret;
} }
long long exec_serial_rule_begin(redisContext* c, size_t rule_num, size_t renew_rule_num, long long exec_serial_rule_begin(redisContext* c, size_t rule_num,
int *renew_allowed, long long *transaction_version) size_t renew_rule_num, int *renew_allowed,
long long *transaction_version)
{ {
int ret = -1; int ret = -1;
redisReply *data_reply = NULL; redisReply *data_reply = NULL;
@@ -878,8 +939,9 @@ const char* lua_exec_done=
"redis.call(\'del\', KEYS[4]);" "redis.call(\'del\', KEYS[4]);"
"redis.call(\'zadd\', KEYS[3], ARGV[1], maat_version);" "redis.call(\'zadd\', KEYS[3], ARGV[1], maat_version);"
"return maat_version;"; "return maat_version;";
redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list, long long server_time, redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list,
int renew_allowed, struct expected_reply *expect_reply, size_t *cnt) long long server_time, int renew_allowed,
struct expected_reply *expect_reply, size_t *cnt)
{ {
redisReply *data_reply = NULL; redisReply *data_reply = NULL;
@@ -907,8 +969,10 @@ redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list,
return data_reply; return data_reply;
} }
void exec_serial_rule(redisContext *c, const char *transaction_list, struct serial_rule *s_rule, size_t rule_num, void exec_serial_rule(redisContext *c, const char *transaction_list,
struct expected_reply *expect_reply, size_t *cnt, size_t offset, int renew_allowed) struct serial_rule *s_rule, size_t rule_num,
struct expected_reply *expect_reply, size_t *cnt,
size_t offset, int renew_allowed)
{ {
size_t i = 0; size_t i = 0;
size_t append_cmd_cnt = 0; size_t append_cmd_cnt = 0;
@@ -1053,8 +1117,9 @@ int mr_operation_success(redisReply *actual_reply, struct expected_reply *expect
return 0; return 0;
} }
int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
long long server_time, struct log_handle *logger) size_t serial_rule_num, long long server_time,
struct log_handle *logger)
{ {
size_t i = 0; size_t i = 0;
size_t rule_seq = 0; size_t rule_seq = 0;
@@ -1079,7 +1144,8 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri
} }
} }
int ret = exec_serial_rule_begin(c, serial_rule_num, renew_num, &renew_allowed, &transaction_version); int ret = exec_serial_rule_begin(c, serial_rule_num, renew_num, &renew_allowed,
&transaction_version);
//Preconditions for transaction are not satisfied. //Preconditions for transaction are not satisfied.
if (ret != 0) { if (ret != 0) {
success_cnt = -1; success_cnt = -1;
@@ -1087,18 +1153,20 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri
} }
if (transaction_version > 0) { if (transaction_version > 0) {
snprintf(transaction_list, sizeof(transaction_list), "MAAT_TRANSACTION_%lld", transaction_version); snprintf(transaction_list, sizeof(transaction_list), "MAAT_TRANSACTION_%lld",
transaction_version);
} }
while (success_cnt < serial_rule_num) { while (success_cnt < serial_rule_num) {
size_t batch_cnt = MIN(serial_rule_num - success_cnt, max_redis_batch); size_t batch_cnt = MIN(serial_rule_num - success_cnt, max_redis_batch);
exec_serial_rule(c, transaction_list, s_rule + success_cnt, batch_cnt, expected_reply, &multi_cmd_cnt, exec_serial_rule(c, transaction_list, s_rule + success_cnt, batch_cnt,
success_cnt, renew_allowed); expected_reply, &multi_cmd_cnt, success_cnt, renew_allowed);
assert(multi_cmd_cnt<max_multi_cmd_num); assert(multi_cmd_cnt<max_multi_cmd_num);
success_cnt+=batch_cnt; success_cnt+=batch_cnt;
} }
transaction_reply = exec_serial_rule_end(c, transaction_list, server_time, renew_allowed, expected_reply, &multi_cmd_cnt); transaction_reply = exec_serial_rule_end(c, transaction_list, server_time, renew_allowed,
expected_reply, &multi_cmd_cnt);
if (1 == mr_transaction_success(transaction_reply)) { if (1 == mr_transaction_success(transaction_reply)) {
assert(transaction_reply->elements == multi_cmd_cnt); assert(transaction_reply->elements == multi_cmd_cnt);
for (i = 0; i < multi_cmd_cnt; i++) { for (i = 0; i < multi_cmd_cnt; i++) {
@@ -1138,8 +1206,10 @@ error_out:
if (renew_num > 0 && renew_allowed != 1) { if (renew_num > 0 && renew_allowed != 1) {
for (i = 0; i < (unsigned int)serial_rule_num; i++) { for (i = 0; i < (unsigned int)serial_rule_num; i++) {
if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) { if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) {
log_error(logger, MODULE_REDIS_MONITOR, "%s %s %lu is not allowed due to lock contention", log_error(logger, MODULE_REDIS_MONITOR,
mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, s_rule[i].rule_id); "%s %s %lu is not allowed due to lock contention",
mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name,
s_rule[i].rule_id);
} }
} }
@@ -1171,11 +1241,11 @@ void cleanup_update_status(redisContext *c, struct log_handle *logger)
int append_cmd_cnt = 0; int append_cmd_cnt = 0;
redisAppendCommand(c, "ZRANGEBYSCORE %s -inf %lld", redisAppendCommand(c, "ZRANGEBYSCORE %s -inf %lld",
mr_version_sset, server_time - MAAT_REDIS_SYNC_TIME); mr_version_sset, server_time - MAAT_REDIS_SYNC_TIME);
append_cmd_cnt++; append_cmd_cnt++;
redisAppendCommand(c, "ZREMRANGEBYSCORE %s -inf %lld", redisAppendCommand(c, "ZREMRANGEBYSCORE %s -inf %lld",
mr_version_sset,server_time - MAAT_REDIS_SYNC_TIME); mr_version_sset,server_time - MAAT_REDIS_SYNC_TIME);
append_cmd_cnt++; append_cmd_cnt++;
//consume reply "OK" and "QUEUED". //consume reply "OK" and "QUEUED".
@@ -1207,8 +1277,9 @@ void cleanup_update_status(redisContext *c, struct log_handle *logger)
reply = NULL; reply = NULL;
//To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally. //To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally.
reply = maat_cmd_wrap_redis_command(c, "ZREMRANGEBYSCORE %s %lld %lld", mr_status_sset, reply = maat_cmd_wrap_redis_command(c, "ZREMRANGEBYSCORE %s %lld %lld",
version_lower_bound, version_upper_bound); mr_status_sset, version_lower_bound,
version_upper_bound);
entry_num = maat_cmd_read_redis_integer(reply); entry_num = maat_cmd_read_redis_integer(reply);
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
@@ -1232,7 +1303,8 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger)
return; return;
} }
redisReply *data_reply= maat_cmd_wrap_redis_command(c, "ZRANGEBYSCORE %s -inf %lld", mr_expire_sset, server_time); redisReply *data_reply= maat_cmd_wrap_redis_command(c, "ZRANGEBYSCORE %s -inf %lld",
mr_expire_sset, server_time);
if (data_reply->type != REDIS_REPLY_ARRAY || 0 == data_reply->elements) { if (data_reply->type != REDIS_REPLY_ARRAY || 0 == data_reply->elements) {
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply = NULL; data_reply = NULL;
@@ -1244,7 +1316,8 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger)
for (size_t i = 0; i < s_rule_num; i++) { for (size_t i = 0; i < s_rule_num; i++) {
s_rule[i].op = MAAT_OP_DEL; s_rule[i].op = MAAT_OP_DEL;
ret = sscanf(data_reply->element[i]->str, "%[^,],%ld", s_rule[i].table_name, &(s_rule[i].rule_id)); ret = sscanf(data_reply->element[i]->str, "%[^,],%ld",
s_rule[i].table_name, &(s_rule[i].rule_id));
assert(ret == 2); assert(ret == 2);
} }
freeReplyObject(data_reply); freeReplyObject(data_reply);
@@ -1254,9 +1327,11 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger)
if (success_cnt < 0) { if (success_cnt < 0) {
log_error(logger, MODULE_REDIS_MONITOR, "maat_cmd_write_rule failed."); log_error(logger, MODULE_REDIS_MONITOR, "maat_cmd_write_rule failed.");
} else if (success_cnt == (int)s_rule_num) { } else if (success_cnt == (int)s_rule_num) {
log_info(logger, MODULE_REDIS_MONITOR, "Succesfully expired %zu rules in Redis", s_rule_num); log_info(logger, MODULE_REDIS_MONITOR,
"Succesfully expired %zu rules in Redis", s_rule_num);
} else { } else {
log_error(logger, MODULE_REDIS_MONITOR, "Failed to expired %d of %zu rules in Redis, try later", log_error(logger, MODULE_REDIS_MONITOR,
"Failed to expired %d of %zu rules in Redis, try later",
s_rule_num - success_cnt, s_rule_num); s_rule_num - success_cnt, s_rule_num);
} }
@@ -1301,7 +1376,9 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
} }
log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Reconnecting..."); log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Reconnecting...");
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip,
mr_ctx->redis_port,
mr_ctx->redis_db,
maat_instance->logger); maat_instance->logger);
if (NULL == mr_ctx->read_ctx) { if (NULL == mr_ctx->read_ctx) {
return; return;
@@ -1312,11 +1389,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
struct serial_rule *rule_list = NULL; struct serial_rule *rule_list = NULL;
long long new_version = 0; long long new_version = 0;
int update_type = CM_UPDATE_TYPE_INC; int update_type = MAAT_UPDATE_TYPE_INC;
int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, maat_instance->load_specific_version, int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version,
&new_version, maat_instance->tbl_mgr, &rule_list, maat_instance->load_specific_version,
&update_type, maat_instance->cumulative_update_off, &new_version, maat_instance->tbl_mgr,
&rule_list, &update_type,
maat_instance->cumulative_update_off,
maat_instance->logger); maat_instance->logger);
//redis communication error //redis communication error
if (rule_num < 0) { if (rule_num < 0) {
@@ -1327,12 +1406,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
maat_instance->load_specific_version = 0;//only valid for one time. maat_instance->load_specific_version = 0;//only valid for one time.
//error or nothing changed //error or nothing changed
if (0 == rule_num && update_type == CM_UPDATE_TYPE_INC) { if (0 == rule_num && update_type == MAAT_UPDATE_TYPE_INC) {
return; return;
} }
if (rule_num > 0) { if (rule_num > 0) {
ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_instance->logger); ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num,
0, maat_instance->logger);
//redis communication error //redis communication error
if (ret < 0) { if (ret < 0) {
redisFree(mr_ctx->read_ctx); redisFree(mr_ctx->read_ctx);
@@ -1354,15 +1434,19 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
goto clean_up; goto clean_up;
} }
ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num, maat_instance, maat_instance->foreign_cont_dir); ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num,
maat_instance, maat_instance->foreign_cont_dir);
if (ret > 0) { if (ret > 0) {
maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_instance->logger); maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0,
maat_instance->logger);
} }
} }
start_fn(new_version, update_type, u_param); start_fn(new_version, update_type, u_param);
log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Start %s update: %lld -> %lld (%d entries)", log_info(maat_instance->logger, MODULE_REDIS_MONITOR,
update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num); "Start %s update: %lld -> %lld (%d entries)",
update_type == MAAT_UPDATE_TYPE_INC ? "INC" : "FULL",
version, new_version, rule_num);
for (i = 0; i < rule_num; i++) { for (i = 0; i < rule_num; i++) {
if (NULL == rule_list[i].table_line) { if (NULL == rule_list[i].table_line) {
@@ -1385,7 +1469,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
ret = invalidate_line(rule_list[i].table_line, table_type, valid_column); ret = invalidate_line(rule_list[i].table_line, table_type, valid_column);
if (ret < 0) { if (ret < 0) {
log_error(maat_instance->logger, MODULE_REDIS_MONITOR, log_error(maat_instance->logger, MODULE_REDIS_MONITOR,
"Invalidate line failed, invaid format %s", rule_list[i].table_line); "Invalidate line failed, invaid format %s",
rule_list[i].table_line);
continue; continue;
} }
} }

81
src/maat_rule.cpp
View File

@@ -17,7 +17,6 @@
#include <sys/stat.h> #include <sys/stat.h>
#include <assert.h> #include <assert.h>
#include "utils.h"
#include "json2iris.h" #include "json2iris.h"
#include "log/log.h" #include "log/log.h"
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
@@ -57,18 +56,18 @@ void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *)
static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, size_t n_accept_tag) static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, size_t n_accept_tag)
{ {
if (NULL == tag_obj || NULL == accept_tags) { if (NULL == tag_obj || NULL == accept_tags) {
return -1; return TAG_MATCH_ERR;
} }
cJSON *tab_name_obj = cJSON_GetObjectItem(tag_obj, "tag"); cJSON *tab_name_obj = cJSON_GetObjectItem(tag_obj, "tag");
if (NULL == tab_name_obj || tab_name_obj->type != cJSON_String) { if (NULL == tab_name_obj || tab_name_obj->type != cJSON_String) {
return -1; return TAG_MATCH_ERR;
} }
const char *tag_name = tab_name_obj->valuestring; const char *tag_name = tab_name_obj->valuestring;
cJSON *tag_vals_array = cJSON_GetObjectItem(tag_obj, "value"); cJSON *tag_vals_array = cJSON_GetObjectItem(tag_obj, "value");
if (NULL == tag_vals_array || tag_vals_array->type != cJSON_Array) { if (NULL == tag_vals_array || tag_vals_array->type != cJSON_Array) {
return -1; return TAG_MATCH_ERR;
} }
int name_matched = 0; int name_matched = 0;
@@ -82,7 +81,7 @@ static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags,
for (int j = 0; j < n_val; j++) { for (int j = 0; j < n_val; j++) {
cJSON *tag_val_obj = cJSON_GetArrayItem(tag_vals_array, j); cJSON *tag_val_obj = cJSON_GetArrayItem(tag_vals_array, j);
if (NULL == tag_val_obj || tag_val_obj->type != cJSON_String) { if (NULL == tag_val_obj || tag_val_obj->type != cJSON_String) {
return -1; return TAG_MATCH_ERR;
} }
const char *tag_val = tag_val_obj->valuestring; const char *tag_val = tag_val_obj->valuestring;
@@ -95,16 +94,16 @@ static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags,
//make sure the overlap is ended with a '/' //make sure the overlap is ended with a '/'
if (0 == strncmp(accept_tags[i].tag_val, tag_val, strlen(tag_val)) && if (0 == strncmp(accept_tags[i].tag_val, tag_val, strlen(tag_val)) &&
(strlen(accept_tags[i].tag_val) == strlen(tag_val) || accept_tags[i].tag_val[strlen(tag_val)] == '/')) { (strlen(accept_tags[i].tag_val) == strlen(tag_val) || accept_tags[i].tag_val[strlen(tag_val)] == '/')) {
return 1; return TAG_MATCH_MATCHED;
} }
} }
} }
//no matched name is considered as a //no matched name is considered as a
if (name_matched > 0) { if (name_matched > 0) {
return 0; return TAG_MATCH_UNMATCHED;
} else { } else {
return 1; return TAG_MATCH_MATCHED;
} }
} }
@@ -117,12 +116,12 @@ static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_ta
for (int i = 0; i < n_tag; i++) { for (int i = 0; i < n_tag; i++) {
cJSON *tag_obj = cJSON_GetArrayItem(tag_set, i); cJSON *tag_obj = cJSON_GetArrayItem(tag_set, i);
if (NULL == tag_obj || tag_obj->type != cJSON_Object) { if (NULL == tag_obj || tag_obj->type != cJSON_Object) {
goto error; return TAG_MATCH_ERR;
} }
int ret = compare_each_tag(tag_obj, accept_tags, n_accept_tag); int ret = compare_each_tag(tag_obj, accept_tags, n_accept_tag);
if (ret < 0) { if (ret < 0) {
return -1; return TAG_MATCH_ERR;
} }
if(1 == ret) { if(1 == ret) {
@@ -131,13 +130,10 @@ static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_ta
} }
if (matched == n_tag) { if (matched == n_tag) {
return 1; return TAG_MATCH_MATCHED;
} else { } else {
return 0; return TAG_MATCH_UNMATCHED;
} }
error:
return -1;
} }
//@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]} //@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]}
@@ -176,7 +172,7 @@ size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_
//@return 1 on match, 0 on not match, -1 on error. //@return 1 on match, 0 on not match, -1 on error.
int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag) int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag)
{ {
int ret = -1; int ret = TAG_MATCH_ERR;
int n_set = 0; int n_set = 0;
cJSON *tag_set = NULL; cJSON *tag_set = NULL;
cJSON *tag_set_array = NULL; cJSON *tag_set_array = NULL;
@@ -200,7 +196,7 @@ int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, si
ret = compare_each_tag_set(tag_set, accept_tags, n_accept_tag); ret = compare_each_tag_set(tag_set, accept_tags, n_accept_tag);
//match or error occurs. //match or error occurs.
if (ret != 0) { if (ret != TAG_MATCH_UNMATCHED) {
break; break;
} }
} }
@@ -280,25 +276,11 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt)
FREE(maat_rt); FREE(maat_rt);
} }
int maat_runtime_updating_flag(struct maat_runtime *maat_rt)
{
int flag = -1;
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
flag = table_manager_runtime_updating_flag(maat_rt->ref_tbl_mgr, i);
if (1 == flag) {
return 1;
}
}
return 0;
}
void maat_start_cb(long long new_version, int update_type, void *u_param) void maat_start_cb(long long new_version, int update_type, void *u_param)
{ {
struct maat *maat_instance = (struct maat *)u_param; struct maat *maat_instance = (struct maat *)u_param;
if (update_type == CM_UPDATE_TYPE_FULL) { if (update_type == MAAT_UPDATE_TYPE_FULL) {
maat_instance->creating_maat_rt = maat_runtime_create(new_version, maat_instance); maat_instance->creating_maat_rt = maat_runtime_create(new_version, maat_instance);
} else { } else {
maat_instance->maat_version = new_version; maat_instance->maat_version = new_version;
@@ -390,14 +372,18 @@ void maat_finish_cb(void *u_param)
if (maat_instance->creating_maat_rt != NULL) { if (maat_instance->creating_maat_rt != NULL) {
maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt);
maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger); maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger);
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Full config version %llu load %d entries complete\n", log_info(maat_instance->logger, MODULE_MAAT_RULE,
maat_instance->creating_maat_rt->version, maat_instance->creating_maat_rt->rule_num); "Full config version %llu load %d entries complete",
maat_instance->creating_maat_rt->version,
maat_instance->creating_maat_rt->rule_num);
} else if (maat_instance->maat_rt != NULL) { } else if (maat_instance->maat_rt != NULL) {
maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt); maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt);
maat_instance->maat_rt->version = maat_instance->maat_version; maat_instance->maat_rt->version = maat_instance->maat_version;
maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger); maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger);
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Inc config version %llu load %d entries complete\n", log_info(maat_instance->logger, MODULE_MAAT_RULE,
maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); "Inc config version %llu load %d entries complete",
maat_instance->maat_rt->version,
maat_instance->maat_rt->rule_num);
} }
} }
@@ -420,7 +406,8 @@ void *rule_monitor_loop(void *arg)
pthread_mutex_lock(&(maat_instance->background_update_mutex)); pthread_mutex_lock(&(maat_instance->background_update_mutex));
/* if deferred load on */ /* if deferred load on */
if (maat_instance->deferred_load != 0) { if (maat_instance->deferred_load != 0) {
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Deferred Loading ON, updating in %s", __func__); log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Deferred Loading ON, updating in %s", __func__);
maat_read_full_config(maat_instance); maat_read_full_config(maat_instance);
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
@@ -478,13 +465,16 @@ void *rule_monitor_loop(void *arg)
if (old_maat_rt != NULL) { if (old_maat_rt != NULL) {
if (maat_instance->maat_rt->version > old_maat_rt->version) { if (maat_instance->maat_rt->version > old_maat_rt->version) {
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Maat version updated %lld -> %lld\n", log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Maat version updated %lld -> %lld\n",
old_maat_rt->version, maat_instance->maat_rt->version); old_maat_rt->version, maat_instance->maat_rt->version);
} else { } else {
log_info(maat_instance->logger, MODULE_MAAT_RULE, "Maat version roll back %lld -> %lld\n", log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Maat version roll back %lld -> %lld\n",
old_maat_rt->version, maat_instance->maat_rt->version); old_maat_rt->version, maat_instance->maat_rt->version);
} }
maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy); maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt,
(void (*)(void*))maat_runtime_destroy);
} }
maat_instance->creating_maat_rt = NULL; maat_instance->creating_maat_rt = NULL;
@@ -493,12 +483,11 @@ void *rule_monitor_loop(void *arg)
} }
if (maat_instance->maat_rt != NULL) { if (maat_instance->maat_rt != NULL) {
int updating_flag = maat_runtime_updating_flag(maat_instance->maat_rt);
time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time; time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time;
if ((updating_flag > 0) && (time_window >= maat_instance->rule_effect_interval_ms / 1000)) { if (time_window >= maat_instance->rule_effect_interval_ms / 1000) {
maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger); maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger);
log_info(maat_instance->logger,MODULE_MAAT_RULE, log_info(maat_instance->logger, MODULE_MAAT_RULE,
"Actual update config version %u, %d entries load to rulescan after postpone.", "Actual update config version %u, %d entries load to rulescan after postpone.",
maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num);
} }
@@ -514,6 +503,12 @@ void *rule_monitor_loop(void *arg)
maat_garbage_bin_free(maat_instance->garbage_bin); maat_garbage_bin_free(maat_instance->garbage_bin);
table_manager_destroy(maat_instance->tbl_mgr); table_manager_destroy(maat_instance->tbl_mgr);
alignment_int64_array_free(maat_instance->outer_state_cnt);
alignment_int64_array_free(maat_instance->compile_state_cnt);
alignment_int64_array_free(maat_instance->thread_call_cnt);
alignment_int64_array_free(maat_instance->hit_cnt);
alignment_int64_array_free(maat_instance->not_grp_hit_cnt);
if (maat_instance->input_mode == DATA_SOURCE_REDIS) { if (maat_instance->input_mode == DATA_SOURCE_REDIS) {
if (maat_instance->mr_ctx.read_ctx != NULL) { if (maat_instance->mr_ctx.read_ctx != NULL) {
redisFree(maat_instance->mr_ctx.read_ctx); redisFree(maat_instance->mr_ctx.read_ctx);

174
src/maat_table.cpp
View File

@@ -12,7 +12,6 @@
#include <assert.h> #include <assert.h>
#include "log/log.h" #include "log/log.h"
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_table.h" #include "maat_table.h"
#include "maat_rule.h" #include "maat_rule.h"
@@ -26,7 +25,7 @@
#include "maat_ip_plugin.h" #include "maat_ip_plugin.h"
#include "maat_virtual.h" #include "maat_virtual.h"
#define MODULE_TABLE module_name_str("maat.table") #define MODULE_TABLE module_name_str("maat.table")
struct table_item { struct table_item {
enum table_type table_type; enum table_type table_type;
@@ -50,21 +49,28 @@ struct table_manager {
size_t n_accept_tag; size_t n_accept_tag;
int default_compile_table_id; int default_compile_table_id;
int g2g_table_id;
struct maat_kv_store *tablename2id_map; struct maat_kv_store *tablename2id_map;
struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger; struct log_handle *logger;
}; };
struct table_operations { struct table_operations {
enum table_type type; enum table_type type;
void *(*new_schema)(cJSON *json, const char *table_name, struct log_handle *logger); void *(*new_schema)(cJSON *json, struct table_manager *tbl_mgr, const char *table_name,
struct log_handle *logger);
void (*free_schema)(void *schema); void (*free_schema)(void *schema);
void *(*new_runtime)(void *schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void *(*new_runtime)(void *schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void (*free_runtime)(void *runtime); void (*free_runtime)(void *runtime);
int (*update_runtime)(void *runtime, void *schema, const char *line, int valid_column); int (*update_runtime)(void *runtime, void *schema, const char *line, int valid_column);
int (*commit_runtime)(void *runtime); int (*commit_runtime)(void *runtime);
int (*runtime_updating_flag)(void *runtime);
}; };
struct table_operations table_ops[TABLE_TYPE_MAX] = { struct table_operations table_ops[TABLE_TYPE_MAX] = {
@@ -75,8 +81,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = expr_runtime_new, .new_runtime = expr_runtime_new,
.free_runtime = expr_runtime_free, .free_runtime = expr_runtime_free,
.update_runtime = expr_runtime_update, .update_runtime = expr_runtime_update,
.commit_runtime = expr_runtime_commit, .commit_runtime = expr_runtime_commit
.runtime_updating_flag = expr_runtime_updating_flag
}, },
{ {
.type = TABLE_TYPE_EXPR_PLUS, .type = TABLE_TYPE_EXPR_PLUS,
@@ -85,18 +90,16 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = expr_runtime_new, .new_runtime = expr_runtime_new,
.free_runtime = expr_runtime_free, .free_runtime = expr_runtime_free,
.update_runtime = expr_runtime_update, .update_runtime = expr_runtime_update,
.commit_runtime = expr_runtime_commit, .commit_runtime = expr_runtime_commit
.runtime_updating_flag = expr_runtime_updating_flag
}, },
{ {
.type = TABLE_TYPE_IP_PLUS, .type = TABLE_TYPE_IP_PLUS,
.new_schema = ip_plus_schema_new, .new_schema = ip_schema_new,
.free_schema = ip_plus_schema_free, .free_schema = ip_schema_free,
.new_runtime = ip_plus_runtime_new, .new_runtime = ip_runtime_new,
.free_runtime = ip_plus_runtime_free, .free_runtime = ip_runtime_free,
.update_runtime = ip_plus_runtime_update, .update_runtime = ip_runtime_update,
.commit_runtime = ip_plus_runtime_commit, .commit_runtime = ip_runtime_commit
.runtime_updating_flag = ip_plus_runtime_updating_flag
}, },
{ {
.type = TABLE_TYPE_INTERVAL, .type = TABLE_TYPE_INTERVAL,
@@ -105,8 +108,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
.commit_runtime = NULL, .commit_runtime = NULL
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_INTERVAL_PLUS, .type = TABLE_TYPE_INTERVAL_PLUS,
@@ -115,8 +117,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
.commit_runtime = NULL, .commit_runtime = NULL
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_DIGEST, .type = TABLE_TYPE_DIGEST,
@@ -125,8 +126,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
.commit_runtime = NULL, .commit_runtime = NULL
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_SIMILARITY, .type = TABLE_TYPE_SIMILARITY,
@@ -135,8 +135,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
.commit_runtime = NULL, .commit_runtime = NULL
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_CONJUNCTION, .type = TABLE_TYPE_CONJUNCTION,
@@ -145,8 +144,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
.commit_runtime = NULL, .commit_runtime = NULL
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_PLUGIN, .type = TABLE_TYPE_PLUGIN,
@@ -155,8 +153,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = plugin_runtime_new, .new_runtime = plugin_runtime_new,
.free_runtime = plugin_runtime_free, .free_runtime = plugin_runtime_free,
.update_runtime = plugin_runtime_update, .update_runtime = plugin_runtime_update,
.commit_runtime = plugin_runtime_commit, .commit_runtime = plugin_runtime_commit
.runtime_updating_flag = plugin_runtime_updating_flag
}, },
{ {
.type = TABLE_TYPE_IP_PLUGIN, .type = TABLE_TYPE_IP_PLUGIN,
@@ -165,8 +162,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = ip_plugin_runtime_new, .new_runtime = ip_plugin_runtime_new,
.free_runtime = ip_plugin_runtime_free, .free_runtime = ip_plugin_runtime_free,
.update_runtime = ip_plugin_runtime_update, .update_runtime = ip_plugin_runtime_update,
.commit_runtime = ip_plugin_runtime_commit, .commit_runtime = ip_plugin_runtime_commit
.runtime_updating_flag = ip_plugin_runtime_updating_flag
}, },
{ {
.type = TABLE_TYPE_FQDN_PLUGIN, .type = TABLE_TYPE_FQDN_PLUGIN,
@@ -202,8 +198,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = compile_runtime_new, .new_runtime = compile_runtime_new,
.free_runtime = compile_runtime_free, .free_runtime = compile_runtime_free,
.update_runtime = compile_runtime_update, .update_runtime = compile_runtime_update,
.commit_runtime = compile_runtime_commit, .commit_runtime = compile_runtime_commit
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_GROUP2GROUP, .type = TABLE_TYPE_GROUP2GROUP,
@@ -212,8 +207,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = group2group_runtime_new, .new_runtime = group2group_runtime_new,
.free_runtime = group2group_runtime_free, .free_runtime = group2group_runtime_free,
.update_runtime = group2group_runtime_update, .update_runtime = group2group_runtime_update,
.commit_runtime = group2group_runtime_commit, .commit_runtime = group2group_runtime_commit
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_GROUP2COMPILE, .type = TABLE_TYPE_GROUP2COMPILE,
@@ -222,18 +216,19 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = group2compile_runtime_new, .new_runtime = group2compile_runtime_new,
.free_runtime = group2compile_runtime_free, .free_runtime = group2compile_runtime_free,
.update_runtime = group2compile_runtime_update, .update_runtime = group2compile_runtime_update,
.commit_runtime = NULL, .commit_runtime = NULL
.runtime_updating_flag = NULL
} }
}; };
void *maat_table_schema_new(cJSON *json, const char *table_name, enum table_type table_type, void *maat_table_schema_new(cJSON *json, const char *table_name,
enum table_type table_type,
struct table_manager *tbl_mgr,
struct log_handle *logger) struct log_handle *logger)
{ {
void *schema = NULL; void *schema = NULL;
if (table_ops[table_type].new_schema != NULL) { if (table_ops[table_type].new_schema != NULL) {
schema = table_ops[table_type].new_schema(json, table_name, logger); schema = table_ops[table_type].new_schema(json, tbl_mgr, table_name, logger);
} }
return schema; return schema;
@@ -378,8 +373,9 @@ void maat_table_free(struct maat_table *maat_tbl)
FREE(maat_tbl); FREE(maat_tbl);
} }
struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags, struct table_manager *
struct log_handle *logger) table_manager_create(const char *table_info_path, const char *accept_tags,
struct maat_garbage_bin *garbage_bin, struct log_handle *logger)
{ {
if (NULL == table_info_path) { if (NULL == table_info_path) {
return NULL; return NULL;
@@ -389,7 +385,8 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
size_t json_buff_sz = 0; size_t json_buff_sz = 0;
int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz); int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_TABLE, "Maat read table info %s error.", table_info_path); log_error(logger, MODULE_TABLE,
"Maat read table info %s error.", table_info_path);
return NULL; return NULL;
} }
@@ -397,14 +394,16 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
cJSON *json = NULL; cJSON *json = NULL;
root = cJSON_Parse((const char *)json_buff); root = cJSON_Parse((const char *)json_buff);
if (!root) { if (!root) {
log_error(logger, MODULE_TABLE, "Error before: %-200.200s", cJSON_GetErrorPtr()); log_error(logger, MODULE_TABLE,
"Error before: %-200.200s", cJSON_GetErrorPtr());
FREE(json_buff); FREE(json_buff);
return NULL; return NULL;
} }
int json_array_size = cJSON_GetArraySize(root); int json_array_size = cJSON_GetArraySize(root);
if (json_array_size <= 0) { if (json_array_size <= 0) {
log_error(logger, MODULE_TABLE, "invalid json content in %s", table_info_path); log_error(logger, MODULE_TABLE,
"invalid json content in %s", table_info_path);
free(json_buff); free(json_buff);
return NULL; return NULL;
} }
@@ -416,8 +415,11 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger); tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger);
tbl_mgr->logger = logger; tbl_mgr->logger = logger;
tbl_mgr->tablename2id_map = maat_kv_store_new(); tbl_mgr->tablename2id_map = maat_kv_store_new();
tbl_mgr->district_map = maat_kv_store_new();
tbl_mgr->ref_garbage_bin = garbage_bin;
int default_compile_table_id = MAX_TABLE_NUM; int default_compile_table_id = MAX_TABLE_NUM;
int g2g_table_id = MAX_TABLE_NUM;
for (int i = 0; i < json_array_size; i++) { for (int i = 0; i < json_array_size; i++) {
json = cJSON_GetArrayItem(root, i); json = cJSON_GetArrayItem(root, i);
@@ -437,10 +439,11 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
continue; continue;
} }
maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type, logger); maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name,
maat_tbl->table_type, tbl_mgr, logger);
if (NULL == maat_tbl->schema) { if (NULL == maat_tbl->schema) {
log_error(logger, MODULE_TABLE, "Maat table schema new failed, table_name:%s", log_error(logger, MODULE_TABLE,
maat_tbl->table_name); "Maat table schema new failed, table_name:%s", maat_tbl->table_name);
maat_table_free(maat_tbl); maat_table_free(maat_tbl);
continue; continue;
} }
@@ -451,14 +454,23 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
} }
} }
if (maat_tbl->table_type == TABLE_TYPE_GROUP2GROUP) {
g2g_table_id = maat_tbl->table_id;
}
tbl_mgr->tbl[maat_tbl->table_id] = maat_tbl; tbl_mgr->tbl[maat_tbl->table_id] = maat_tbl;
tbl_mgr->n_table++; tbl_mgr->n_table++;
} }
} }
assert(default_compile_table_id != MAX_TABLE_NUM); assert(default_compile_table_id != MAX_TABLE_NUM);
assert(g2g_table_id != MAX_TABLE_NUM);
tbl_mgr->default_compile_table_id = default_compile_table_id; tbl_mgr->default_compile_table_id = default_compile_table_id;
tbl_mgr->g2g_table_id = g2g_table_id;
log_info(logger, MODULE_TABLE, "default compile table id: %d", default_compile_table_id); log_info(logger, MODULE_TABLE, "default compile table id: %d", default_compile_table_id);
log_info(logger, MODULE_TABLE, "group2group table id: %d", g2g_table_id);
maat_kv_store_free(reserved_word_map); maat_kv_store_free(reserved_word_map);
cJSON_Delete(root); cJSON_Delete(root);
@@ -490,7 +502,6 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n
assert(tbl_mgr->n_table != 0); assert(tbl_mgr->n_table != 0);
size_t i = 0; size_t i = 0;
int g2g_group_id = MAX_TABLE_NUM;
enum table_type table_type = TABLE_TYPE_MAX; enum table_type table_type = TABLE_TYPE_MAX;
for (i = 0; i < MAX_TABLE_NUM; i++) { for (i = 0; i < MAX_TABLE_NUM; i++) {
@@ -500,17 +511,11 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n
} }
table_type = table_manager_get_table_type(tbl_mgr, i); table_type = table_manager_get_table_type(tbl_mgr, i);
if (table_type == TABLE_TYPE_GROUP2GROUP) {
g2g_group_id = i;
}
assert(NULL == tbl_mgr->tbl[i]->runtime); assert(NULL == tbl_mgr->tbl[i]->runtime);
tbl_mgr->tbl[i]->runtime = maat_table_runtime_new(schema, table_type, max_thread_num, tbl_mgr->tbl[i]->runtime = maat_table_runtime_new(schema, table_type, max_thread_num,
garbage_bin, tbl_mgr->logger); garbage_bin, tbl_mgr->logger);
} }
assert(g2g_group_id != MAX_TABLE_NUM);
/* group2compile runtime depends on associated compile runtime, /* group2compile runtime depends on associated compile runtime,
must make sure associated compile runtime already exist */ must make sure associated compile runtime already exist */
for (i = 0; i < MAX_TABLE_NUM; i++) { for (i = 0; i < MAX_TABLE_NUM; i++) {
@@ -528,10 +533,14 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n
void *schema = table_manager_get_schema(tbl_mgr, i); void *schema = table_manager_get_schema(tbl_mgr, i);
int associated_compile_table_id = group2compile_associated_compile_table_id(schema); int associated_compile_table_id = group2compile_associated_compile_table_id(schema);
void *compile_rt = table_manager_get_runtime(tbl_mgr, associated_compile_table_id); void *compile_rt = table_manager_get_runtime(tbl_mgr, associated_compile_table_id);
int g2g_group_id = table_manager_get_group2group_table_id(tbl_mgr);
void *g2g_rt = table_manager_get_runtime(tbl_mgr, g2g_group_id); void *g2g_rt = table_manager_get_runtime(tbl_mgr, g2g_group_id);
group2compile_runtime_init(runtime, compile_rt, g2g_rt); group2compile_runtime_init(runtime, compile_rt, g2g_rt);
} }
/* new district map */
tbl_mgr->district_map = maat_kv_store_new();
return 0; return 0;
} }
@@ -551,6 +560,9 @@ void table_manager_runtime_destroy(struct table_manager *tbl_mgr)
maat_table_runtime_free(runtime, table_type); maat_table_runtime_free(runtime, table_type);
tbl_mgr->tbl[i]->runtime = NULL; tbl_mgr->tbl[i]->runtime = NULL;
} }
/* free district map */
maat_kv_store_free(tbl_mgr->district_map);
} }
void table_manager_destroy(struct table_manager *tbl_mgr) void table_manager_destroy(struct table_manager *tbl_mgr)
@@ -580,6 +592,7 @@ void table_manager_destroy(struct table_manager *tbl_mgr)
FREE(tbl_mgr->accept_tags); FREE(tbl_mgr->accept_tags);
maat_kv_store_free(tbl_mgr->tablename2id_map); maat_kv_store_free(tbl_mgr->tablename2id_map);
FREE(tbl_mgr); FREE(tbl_mgr);
} }
@@ -622,6 +635,11 @@ int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr)
return tbl_mgr->default_compile_table_id; return tbl_mgr->default_compile_table_id;
} }
int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr)
{
return tbl_mgr->g2g_table_id;
}
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id) void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id)
{ {
if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) { if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) {
@@ -635,7 +653,8 @@ void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id)
return tbl_mgr->tbl[table_id]->schema; return tbl_mgr->tbl[table_id]->schema;
} }
struct ex_data_schema *table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id) struct ex_data_schema *
table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id)
{ {
return NULL; return NULL;
} }
@@ -653,9 +672,21 @@ int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id)
return tbl_mgr->tbl[table_id]->valid_column; return tbl_mgr->tbl[table_id]->valid_column;
} }
size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr)
{
return tbl_mgr->n_accept_tag;
}
int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags) int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags)
{ {
return 0; return compare_accept_tag(tags, tbl_mgr->accept_tags, tbl_mgr->n_accept_tag);
}
int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str,
size_t district_str_len, int *district_id)
{
return maat_kv_read_unNull(tbl_mgr->district_map,
district_str, district_str_len, district_id);
} }
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id) void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id)
@@ -671,26 +702,8 @@ void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id)
return tbl_mgr->tbl[table_id]->runtime; return tbl_mgr->tbl[table_id]->runtime;
} }
int table_manager_runtime_updating_flag(struct table_manager *tbl_mgr, int table_id) int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id,
{ const char *line)
if (NULL == tbl_mgr) {
return -1;
}
if (NULL == tbl_mgr->tbl[table_id]) {
return -1;
}
enum table_type table_type = tbl_mgr->tbl[table_id]->table_type;
void *runtime = table_manager_get_runtime(tbl_mgr, table_id);
if (table_ops[table_type].runtime_updating_flag != NULL) {
return table_ops[table_type].runtime_updating_flag(runtime);
}
return -1;
}
int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line)
{ {
void *schema = table_manager_get_schema(tbl_mgr, table_id); void *schema = table_manager_get_schema(tbl_mgr, table_id);
if (NULL == schema) { if (NULL == schema) {
@@ -716,7 +729,16 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, co
return -1; return -1;
} }
return table_ops[table_type].update_runtime(runtime, schema, line, valid_column); int ret = table_ops[table_type].update_runtime(runtime, schema, line, valid_column);
if (tbl_mgr->tmp_district_map != NULL) {
struct maat_kv_store *tmp_map = tbl_mgr->district_map;
tbl_mgr->district_map = tbl_mgr->tmp_district_map;
tbl_mgr->tmp_district_map = NULL;
maat_garbage_bagging(tbl_mgr->ref_garbage_bin, tmp_map, (void (*)(void *))maat_kv_store_free);
}
return ret;
} }
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id) void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)

11
src/maat_utils.cpp
View File

@@ -17,7 +17,6 @@
#include <openssl/md5.h> #include <openssl/md5.h>
#include <openssl/evp.h> #include <openssl/evp.h>
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
pid_t gettid() pid_t gettid()
@@ -257,8 +256,8 @@ char *md5_file(const char *filename, char *md5string)
return md5string; return md5string;
} }
int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, size_t *out_sz, int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out,
const char *key, const char *algorithm, int do_encrypt, size_t *out_sz, const char *key, const char *algorithm, int do_encrypt,
char *err_str, size_t err_str_sz) char *err_str, size_t err_str_sz)
{ {
OpenSSL_add_all_algorithms(); OpenSSL_add_all_algorithms();
@@ -325,7 +324,8 @@ error_out:
} }
int decrypt_open(const char* file_name, const char* key, const char* algorithm, int decrypt_open(const char* file_name, const char* key, const char* algorithm,
unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz) unsigned char**pp_out, size_t *out_sz,
char* err_str, size_t err_str_sz)
{ {
size_t file_sz = 0; size_t file_sz = 0;
unsigned char *file_buff = NULL; unsigned char *file_buff = NULL;
@@ -334,7 +334,8 @@ int decrypt_open(const char* file_name, const char* key, const char* algorithm,
return -1; return -1;
} }
ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz); ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm,
0, err_str, err_str_sz);
FREE(file_buff); FREE(file_buff);
return ret; return ret;

11
src/maat_virtual.cpp
View File

@@ -10,7 +10,6 @@
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
#include "maat_kv.h" #include "maat_kv.h"
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "log/log.h" #include "log/log.h"
#include "maat_virtual.h" #include "maat_virtual.h"
@@ -19,19 +18,25 @@
struct virtual_schema { struct virtual_schema {
int physical_table_id[SCAN_TYPE_MAX]; int physical_table_id[SCAN_TYPE_MAX];
int table_id;
struct table_manager *tbl_mgr;
}; };
void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{ {
//size_t read_cnt = 0; //size_t read_cnt = 0;
cJSON *item = cJSON_GetObjectItem(json, "physical_table"); cJSON *item = cJSON_GetObjectItem(json, "physical_table");
if (NULL == item || item->type != cJSON_Array) { if (NULL == item || item->type != cJSON_Array) {
log_error(logger, MODULE_VIRTUAL, "virtual table %s has no physical_table column", table_name); log_error(logger, MODULE_VIRTUAL,
"virtual table %s has no physical_table column", table_name);
return NULL; return NULL;
} }
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
vt_schema->tbl_mgr = tbl_mgr;
return vt_schema; return vt_schema;
#if 0 #if 0
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);

16
src/rcu_hash.cpp
View File

@@ -14,7 +14,7 @@
#include <sys/queue.h> #include <sys/queue.h>
#include "rcu_hash.h" #include "rcu_hash.h"
#include "utils.h" #include "maat_utils.h"
struct rcu_hash_garbage_bag { struct rcu_hash_garbage_bag {
void *garbage; void *garbage;
@@ -68,7 +68,8 @@ size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable)
return htable->garbage_q_len; return htable->garbage_q_len;
} }
void rcu_hash_garbage_bagging(struct rcu_hash_garbage_q* garbage_q, void* garbage, void (* func)(void *)) void rcu_hash_garbage_bagging(struct rcu_hash_garbage_q* garbage_q,
void* garbage, void (* func)(void *))
{ {
struct rcu_hash_garbage_bag *bag = ALLOC(struct rcu_hash_garbage_bag, 1); struct rcu_hash_garbage_bag *bag = ALLOC(struct rcu_hash_garbage_bag, 1);
@@ -171,7 +172,8 @@ void rcu_hash_commit_prepare(struct rcu_hash_table *htable)
htable->is_updating = 1; htable->is_updating = 1;
} }
void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len, void *data) void rcu_hash_add(struct rcu_hash_table *htable, const char *key,
size_t key_len, void *data)
{ {
if (NULL == htable || NULL == key || 0 == key_len) { if (NULL == htable || NULL == key || 0 == key_len) {
return; return;
@@ -227,7 +229,8 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len
} }
if (node != NULL) { if (node != NULL) {
rcu_hash_garbage_bagging(&(htable->garbage_q), node, (void (*)(void*))rcu_hash_node_free); rcu_hash_garbage_bagging(&(htable->garbage_q), node,
(void (*)(void*))rcu_hash_node_free);
htable->garbage_q_len++; htable->garbage_q_len++;
} }
} }
@@ -345,8 +348,3 @@ size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_a
return node_cnt; return node_cnt;
} }
int rcu_hash_updating_flag(struct rcu_hash_table *htable)
{
return htable->is_updating;
}

175
test/maat_framework_gtest.cpp
View File

@@ -1,6 +1,5 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "utils.h"
#include "maat/maat.h" #include "maat/maat.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_utils.h" #include "maat_utils.h"
@@ -9,64 +8,95 @@
#include "json2iris.h" #include "json2iris.h"
#include "maat_config_monitor.h" #include "maat_config_monitor.h"
struct maat *g_maat_instance = NULL; #define MODULE_FRAMEWORK_GTEST module_name_str("maat.framework_gtest")
const char *table_info_path = "./table_info.conf"; const char *table_info_path = "./table_info.conf";
const char *json_path="./maat_json.json"; const char *json_path="./maat_json.json";
const char *json_filename = "maat_json.json"; const char *json_filename = "maat_json.json";
struct maat *g_maat_instance = NULL;
TEST(maat_scan_string, hit_one_expr) { class MaatStringScan : public testing::Test
int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); {
protected:
static void SetUpTestCase() {
table_id = maat_table_get_id(g_maat_instance, "HTTP_URL");
}
static void TearDownTestCase() {
}
static int table_id;
};
int MaatStringScan::table_id;
TEST_F(MaatStringScan, hit_one_expr) {
int table_id = MaatStringScan::table_id;
char scan_data[128] = "hello"; char scan_data[128] = "hello";
int results[5] = {0}; int results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191); EXPECT_EQ(results[0], 191);
struct maat_hit_path hit_path[128] = {0}; struct maat_hit_path hit_path[128] = {0};
int n_read = 0; int n_read = 0;
n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, sizeof(hit_path)); n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, sizeof(hit_path));
EXPECT_NE(n_read, 0);
maat_state_free(&state); maat_state_free(&state);
} }
TEST(maat_scan_string, hit_two_expr) { TEST_F(MaatStringScan, hit_two_expr) {
int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); int table_id = MaatStringScan::table_id;
char data[128] = "should hit aaa bbb"; char data[128] = "should hit aaa bbb";
int results[5] = {0}; int results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 2); EXPECT_EQ(n_hit_result, 2);
EXPECT_EQ(results[0], 28); EXPECT_EQ(results[0], 139);
EXPECT_EQ(results[1], 27); EXPECT_EQ(results[1], 138);
maat_state_free(&state); maat_state_free(&state);
} }
TEST(maat_scan_string, hit_three_expr) { TEST_F(MaatStringScan, hit_three_expr) {
int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); int table_id = MaatStringScan::table_id;
char data[128] = "should hit aaa bbb C#中国"; char data[128] = "should hit aaa bbb C#中国";
int results[5] = {0}; int results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 3); EXPECT_EQ(n_hit_result, 3);
EXPECT_EQ(results[0], 28); EXPECT_EQ(results[0], 139);
EXPECT_EQ(results[1], 27); EXPECT_EQ(results[1], 138);
EXPECT_EQ(results[2], 18); EXPECT_EQ(results[2], 129);
maat_state_free(&state); maat_state_free(&state);
} }
TEST(maat_scan_ipv4, hit_ip_and_port) { class MaatIPScan : public testing::Test
int table_id = maat_table_get_id(g_maat_instance, "IP_PLUS_CONFIG"); {
protected:
static void SetUpTestCase() {
table_id = maat_table_get_id(g_maat_instance, "IP_PLUS_CONFIG");
}
static void TearDownTestCase() {
}
static int table_id;
};
int MaatIPScan::table_id;
TEST_F(MaatIPScan, hit_ip_and_port) {
int table_id = MaatIPScan::table_id;
char ip_str[32] = "192.168.58.19"; char ip_str[32] = "192.168.58.19";
uint32_t sip; uint32_t sip;
int ret = inet_pton(AF_INET, ip_str, &sip); int ret = inet_pton(AF_INET, ip_str, &sip);
@@ -76,20 +106,20 @@ TEST(maat_scan_ipv4, hit_ip_and_port) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 7); EXPECT_EQ(results[0], 7);
maat_state_free(&state); maat_state_free(&state);
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
} }
#if 0
TEST(maat_scan_ipv4, hit_ip_and_port_range) { TEST_F(MaatStringScan, hit_ip_and_port_range) {
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "192.168.50.24"; char ip_str[32] = "192.168.50.24";
uint32_t sip; uint32_t sip;
@@ -100,7 +130,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 4); EXPECT_EQ(results[0], 4);
maat_state_free(&state); maat_state_free(&state);
@@ -109,7 +139,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
n_hit_result = 0; n_hit_result = 0;
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 4); EXPECT_EQ(results[0], 4);
maat_state_free(&state); maat_state_free(&state);
@@ -118,7 +148,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
n_hit_result = 0; n_hit_result = 0;
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
} }
@@ -135,7 +165,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 8); EXPECT_EQ(results[0], 8);
maat_state_free(&state); maat_state_free(&state);
@@ -144,7 +174,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 8); EXPECT_EQ(results[0], 8);
maat_state_free(&state); maat_state_free(&state);
@@ -153,7 +183,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
} }
@@ -170,7 +200,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 50); EXPECT_EQ(results[0], 50);
maat_state_free(&state); maat_state_free(&state);
@@ -179,7 +209,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) {
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
} }
@@ -195,7 +225,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 2); EXPECT_EQ(n_hit_result, 2);
EXPECT_EQ(results[0], 63); EXPECT_EQ(results[0], 63);
EXPECT_EQ(results[1], 67); EXPECT_EQ(results[1], 67);
@@ -203,7 +233,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
} }
@@ -219,14 +249,14 @@ TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 47); EXPECT_EQ(results[0], 47);
maat_state_free(&state); maat_state_free(&state);
state = NULL; state = NULL;
ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
} }
@@ -239,7 +269,7 @@ TEST(maat_scan_string, dynamic_config) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
@@ -255,7 +285,7 @@ TEST(maat_scan_string, dynamic_config) {
sleep(2); sleep(2);
state = NULL; state = NULL;
ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 9999); EXPECT_EQ(results[0], 9999);
maat_state_free(&state); maat_state_free(&state);
@@ -273,7 +303,7 @@ TEST(maat_scan_ip, dynamic_config) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
@@ -289,11 +319,13 @@ TEST(maat_scan_ip, dynamic_config) {
sleep(2); sleep(2);
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 9998); EXPECT_EQ(results[0], 9998);
maat_state_free(&state); maat_state_free(&state);
} }
#endif
int count_line_num_cb(const char *table_name, const char *line, void *u_para) int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{ {
@@ -341,25 +373,22 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
return 0; return 0;
} }
int main(int argc, char ** argv) int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
struct log_handle *logger)
{ {
int ret=0;
::testing::InitGoogleTest(&argc, argv);
char json_iris_path[128] = {0}; char json_iris_path[128] = {0};
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
struct log_handle *logger = log_handle_create("./tmp.log", 0);
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger); redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger);
EXPECT_NE(c, nullptr); if (nullptr == c) {
return -1;
}
redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb"); redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb");
EXPECT_NE(reply, nullptr); if (nullptr == reply) {
return -1;
}
if ((access(json_iris_path, F_OK)) < 0) { if ((access(json_iris_path, F_OK)) < 0) {
char tmp_iris_path[128] = {0}; char tmp_iris_path[128] = {0};
@@ -367,11 +396,15 @@ int main(int argc, char ** argv)
size_t json_buff_sz = 0; size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1); if (ret < 0) {
return -1;
}
ret = json2iris(json_buff, json_filename, c, tmp_iris_path, ret = json2iris(json_buff, json_filename, c, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, logger); sizeof(tmp_iris_path), NULL, NULL, logger);
EXPECT_NE(ret, -1); if (ret < 0) {
return -1;
}
} }
size_t total_line_cnt = 0; size_t total_line_cnt = 0;
@@ -381,7 +414,9 @@ int main(int argc, char ** argv)
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
long long server_time = maat_cmd_redis_server_time_s(c); long long server_time = maat_cmd_redis_server_time_s(c);
EXPECT_NE(server_time, -1); if (server_time < 0) {
return -1;
}
absolute_expire_time = server_time + 300; absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger); config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger);
@@ -399,17 +434,47 @@ int main(int argc, char ** argv)
FREE(s_rule); FREE(s_rule);
redisFree(c); redisFree(c);
return 0;
}
int main(int argc, char ** argv)
{
int ret=0;
::testing::InitGoogleTest(&argc, argv);
struct log_handle *logger = log_handle_create("./maat_framework_gtest.log", 0);
if (NULL == logger) {
printf("create log handle failed.\n");
return -1;
}
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_GTEST, "write config to redis failed.");
log_handle_destroy(logger);
return -1;
}
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, logger); maat_options_set_logger(opts, logger);
g_maat_instance = maat_new(opts, table_info_path); g_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts); maat_options_free(opts);
if (NULL == g_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_GTEST, "create maat instance in MaatStringScan failed.");
log_handle_destroy(logger);
return -1;
}
ret=RUN_ALL_TESTS(); ret=RUN_ALL_TESTS();
log_handle_destroy(g_maat_instance->logger);
maat_free(g_maat_instance); maat_free(g_maat_instance);
log_handle_destroy(logger);
return ret; return ret;
} }

11
test/maat_input_mode_gtest.cpp
View File

@@ -1,4 +1,3 @@
#include "utils.h"
#include "maat/maat.h" #include "maat/maat.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_rule.h" #include "maat_rule.h"
@@ -11,10 +10,6 @@
const char *table_info_path = "./table_info.conf"; const char *table_info_path = "./table_info.conf";
const char *json_filename = "maat_json.json"; const char *json_filename = "maat_json.json";
TEST(EQ_Test, Always_True) {
EXPECT_EQ(1, 1);
}
TEST(json_mode, maat_scan_string) { TEST(json_mode, maat_scan_string) {
char json_iris_path[128] = {0}; char json_iris_path[128] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
@@ -38,7 +33,7 @@ TEST(json_mode, maat_scan_string) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191); EXPECT_EQ(results[0], 191);
@@ -81,7 +76,7 @@ TEST(iris_mode, maat_scan_string) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191); EXPECT_EQ(results[0], 191);
@@ -201,7 +196,7 @@ TEST(redis_mode, maat_scan_string) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191); EXPECT_EQ(results[0], 191);

19
test/rcu_hash_gtest.cpp
View File

@@ -1,5 +1,5 @@
#include "../src/inc_internal/rcu_hash.h" #include "rcu_hash.h"
#include "../include/utils.h" #include "maat_utils.h"
#include <gtest/gtest.h> #include <gtest/gtest.h>
@@ -47,9 +47,6 @@ TEST(rcu_hash_add_one_node, single_thread) {
ret = rcu_hash_list_updating_data(htable, &data_array); ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
ret = rcu_hash_updating_flag(htable);
EXPECT_EQ(ret, 1);
rcu_hash_commit(htable); rcu_hash_commit(htable);
/* find in hash after commit */ /* find in hash after commit */
@@ -69,9 +66,6 @@ TEST(rcu_hash_add_one_node, single_thread) {
ret = rcu_hash_list_updating_data(htable, &data_array); ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
ret = rcu_hash_updating_flag(htable);
EXPECT_EQ(ret, 0);
rcu_hash_free(htable); rcu_hash_free(htable);
} }
@@ -112,9 +106,6 @@ TEST(rcu_hash_add_multi_node, single_thread) {
ret = rcu_hash_list_updating_data(htable, &data_array); ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 2); EXPECT_EQ(ret, 2);
ret = rcu_hash_updating_flag(htable);
EXPECT_EQ(ret, 1);
rcu_hash_commit(htable); rcu_hash_commit(htable);
/* find in hash after commit */ /* find in hash after commit */
@@ -141,9 +132,6 @@ TEST(rcu_hash_add_multi_node, single_thread) {
ret = rcu_hash_list_updating_data(htable, &data_array); ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
ret = rcu_hash_updating_flag(htable);
EXPECT_EQ(ret, 0);
rcu_hash_free(htable); rcu_hash_free(htable);
} }
@@ -181,9 +169,6 @@ TEST(rcu_hash_del_one_node, single_thread) {
ret = rcu_hash_list_updating_data(htable, &data_array); ret = rcu_hash_list_updating_data(htable, &data_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
ret = rcu_hash_updating_flag(htable);
EXPECT_EQ(ret, 1);
rcu_hash_commit(htable); rcu_hash_commit(htable);
/* find in hash after commit */ /* find in hash after commit */

8
test/table_info.conf
View File

@@ -68,7 +68,7 @@
"valid_column":3, "valid_column":3,
"custom": { "custom": {
"group_id":1, "group_id":1,
"superior_group_id":2 "super_group_id":2
} }
}, },
{ {
@@ -125,12 +125,6 @@
"sport_format":7, "sport_format":7,
"sport1":8, "sport1":8,
"sport2":9, "sport2":9,
"daddr_format":10,
"dip1":11,
"dip2":12,
"dport_format":13,
"dport1":14,
"dport2":15,
"proto":16, "proto":16,
"direction":17 "direction":17
} }

5
tools/maat_redis_tool.cpp
View File

@@ -5,7 +5,6 @@
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
#include "utils.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_command.h" #include "maat_command.h"
@@ -70,7 +69,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char
int i = 0; int i = 0;
int ret = 0; int ret = 0;
int line_count = 0; int line_count = 0;
int update_type = CM_UPDATE_TYPE_INC; int update_type = MAAT_UPDATE_TYPE_INC;
long long version = 0; long long version = 0;
const char *cur_table = NULL; const char *cur_table = NULL;
char foreign_files_dir[256] = {0}; char foreign_files_dir[256] = {0};
@@ -95,7 +94,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char
return; return;
} }
assert(update_type == CM_UPDATE_TYPE_FULL); assert(update_type == MAAT_UPDATE_TYPE_FULL);
printf("MAAT Version: %lld, key number: %d\n", version, rule_num); printf("MAAT Version: %lld, key number: %d\n", version, rule_num);
if (0 == rule_num) { if (0 == rule_num) {
goto clean_up; goto clean_up;