gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

framework work well

Browse Source
This commit is contained in:
liuwentan
2022-11-25 16:32:29 +08:00
parent 2a83517894
commit 7e6d131c9e
51 changed files with 3499 additions and 3139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/CMakeLists.txt
View File

@@ -8,24 +8,33 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC maat_api.cpp bool_matcher.cpp adapter_hs.cpp rcu_hash.cpp maat_garbage_collection.cpp)
set(MAAT_SRC maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp
maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp)
set(MAAT_SRC_M main.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp
maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp)
set(LIB_SOURCE_FILES
${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c)
include_directories(${PROJECT_SOURCE_DIR}/include/)
include_directories(${PROJECT_SOURCE_DIR}/deps/)
include_directories(${PROJECT_SOURCE_DIR}/scanner)
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
# Static Library Output
add_library(maat_frame_static STATIC ${MAAT_SRC})
#add_dependencies(maat_frame_static hyperscan_static hyperscan_runtime_static)
add_library(maat_frame_static STATIC ${MAAT_SRC} ${LIB_SOURCE_FILES})
set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX)
set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe)
set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
target_link_libraries(maat_frame_static hyperscan_static)
target_link_libraries(maat_frame_static hyperscan_runtime_static)
target_link_libraries(maat_frame_static adapter-static pthread)
add_executable(main ${MAAT_SRC_M})
target_link_libraries(main maat_frame_static)
# Shared Library Output
#add_library(maat_frame_shared SHARED ${MAAT_SRC})
#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})
#set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX)
#set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe)
#set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1)
@@ -33,7 +42,6 @@ target_link_libraries(maat_frame_static hyperscan_runtime_static)
#set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION})
#target_link_libraries(maat_frame_shared hyperscan_static)
#target_link_libraries(maat_frame_shared hyperscan_runtime_static)
# install
set(CMAKE_INSTALL_PREFIX /opt/MESA/)

534
src/adapter_hs.cpp
View File

@@ -1,534 +0,0 @@
/*
**********************************************************************************************
* File: adapter_hs.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include <stdint.h>
#include <stdio.h>
#include <stddef.h>
#include <hs/hs.h>
#include "sds/sds.h"
#include "adapter_hs.h"
#include "uthash/utarray.h"
#include "uthash/uthash.h"
#include "maat_utils.h"
#include "bool_matcher.h"
struct adpt_hs_compile_data {
unsigned int *ids;
unsigned int *flags;
char **patterns;
size_t *pattern_lens;
unsigned int n_patterns;
};
/* adapter_hs runtime */
struct adapter_hs_runtime {
hs_database_t *literal_db;
hs_database_t *regex_db;
hs_scratch_t **scratchs;
size_t scratch_size;
struct bool_matcher *bm;
};
/* adapter_hs instance */
struct adapter_hs {
size_t nr_worker_threads;
size_t n_expr;
size_t n_patterns;
struct adapter_hs_runtime *hs_rt;
};
struct adapter_hs_stream {
int thread_id;
size_t n_expr;
size_t n_patterns;
hs_stream_t *literal_stream;
hs_stream_t *regex_stream;
struct adapter_hs_runtime *hs_rt;
UT_array *pattern_id_set;
};
int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_threads, int max_pattern_type)
{
hs_database_t *database = NULL;
hs_rt->scratchs = ALLOC(hs_scratch_t *, nr_worker_threads);
if (max_pattern_type == PATTERN_TYPE_STR) {
database = hs_rt->literal_db;
} else {
database = hs_rt->regex_db;
}
if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) {
// log_error("ERROR: Unable to allocate scratch space. Exiting.\n");
hs_free_database(database);
return -1;
}
for (size_t i = 1; i < nr_worker_threads; i++) {
hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]);
if (err != HS_SUCCESS) {
// log_error("Unable to clone scratch prototype");
hs_free_database(database);
return -1;
}
err = hs_scratch_size(hs_rt->scratchs[i], &hs_rt->scratch_size);
if (err != HS_SUCCESS) {
// log_error("Unable to query scratch size");
hs_free_database(database);
return -1;
}
}
return 0;
}
/**
* @brief build hs block database for literal string and regex expression respectively
*
* @retval 0(success) -1(failed)
*/
int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
struct adpt_hs_compile_data *literal_cd,
struct adpt_hs_compile_data *regex_cd,
int scan_mode)
{
hs_error_t err;
hs_compile_error_t *compile_err = NULL;
if (NULL == hs_rt) {
return -1;
}
if (literal_cd != NULL) {
err = hs_compile_lit_multi((const char *const *)literal_cd->patterns, literal_cd->flags,
literal_cd->ids, literal_cd->pattern_lens, literal_cd->n_patterns,
scan_mode, NULL, &hs_rt->literal_db, &compile_err);
if (err != HS_SUCCESS) {
// log_error
if (compile_err) {
printf("compile error: %s", compile_err->message);
}
hs_free_compile_error(compile_err);
goto error;
}
}
if (regex_cd != NULL) {
err = hs_compile_ext_multi((const char *const *)regex_cd->patterns, regex_cd->flags,
regex_cd->ids, NULL, regex_cd->n_patterns,
scan_mode, NULL, &hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) {
// log_error
if (compile_err) {
printf("compile error: %s", compile_err->message);
}
hs_free_compile_error(compile_err);
goto error;
}
}
return 0;
error:
if (hs_rt->literal_db != NULL) {
hs_free_database(hs_rt->literal_db);
hs_rt->literal_db = NULL;
}
if (hs_rt->regex_db != NULL) {
hs_free_database(hs_rt->regex_db);
hs_rt->regex_db = NULL;
}
return -1;
}
struct adpt_hs_compile_data *adpt_hs_compile_data_new(size_t n_patterns)
{
struct adpt_hs_compile_data *hs_cd = ALLOC(struct adpt_hs_compile_data, 1);
hs_cd->patterns = ALLOC(char *, n_patterns);
hs_cd->pattern_lens = ALLOC(size_t, n_patterns);
hs_cd->ids = ALLOC(unsigned int, n_patterns);
hs_cd->flags = ALLOC(unsigned int, n_patterns);
return hs_cd;
}
void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patterns)
{
if (NULL == hs_cd) {
return;
}
if (hs_cd->patterns != NULL) {
for (size_t i = 0; i < n_patterns; i++) {
free(hs_cd->patterns[i]);
}
free(hs_cd->patterns);
free(hs_cd->pattern_lens);
free(hs_cd->ids);
free(hs_cd->flags);
}
free(hs_cd);
}
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array)
{
if ((scan_mode != SCAN_MODE_BLOCK && scan_mode != SCAN_MODE_STREAM) ||
0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) {
/* log_error("input parameters illegal!"); */
return NULL;
}
/* get the sum of pattern */
size_t literal_pattern_num = 0;
size_t regex_pattern_num = 0;
for (size_t i = 0; i < n_expr_array; i++) {
if (expr_array[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
//log_error("the number of patterns in one expression should less than %d\n", MAX_EXPR_PATTERN_NUM);
return NULL;
}
for (size_t j = 0; j < expr_array[i].n_patterns; j++) {
if (expr_array[i].patterns[j].type == PATTERN_TYPE_STR) {
literal_pattern_num++;
} else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) {
regex_pattern_num++;
} else {
/* log_error("unknown pattern type: %d\n", expr_array[i].patterns[j].type); */
return NULL;
}
}
}
struct adpt_hs_compile_data *literal_cd = NULL;
struct adpt_hs_compile_data *regex_cd = NULL;
if (literal_pattern_num > 0) {
literal_cd = adpt_hs_compile_data_new(literal_pattern_num);
}
if (regex_pattern_num > 0) {
regex_cd = adpt_hs_compile_data_new(regex_pattern_num);
}
uint32_t literal_index = 0;
uint32_t regex_index = 0;
uint32_t pattern_id = 0;
/* alloc exprs for bool matcher*/
struct bool_expr *exprs = ALLOC(struct bool_expr, n_expr_array);
/* populate adpt_hs_compile_data and bool_expr */
for (size_t i = 0; i < n_expr_array; i++) {
for (size_t j = 0; j < expr_array[i].n_patterns; j++) {
size_t pat_len = 0;
if (expr_array[i].patterns[j].type == PATTERN_TYPE_STR) {
literal_cd->ids[literal_index] = pattern_id;
literal_cd->flags[literal_index] = HS_FLAG_CASELESS;
pat_len = expr_array[i].patterns[j].pat_len;
literal_cd->pattern_lens[literal_index] = pat_len;
literal_cd->patterns[literal_index] = ALLOC(char, pat_len);
memcpy(literal_cd->patterns[literal_index],
expr_array[i].patterns[j].pat,
expr_array[i].patterns[j].pat_len);
literal_index++;
} else {
regex_cd->ids[regex_index] = pattern_id;
regex_cd->flags[regex_index] = HS_FLAG_CASELESS;
pat_len = expr_array[i].patterns[j].pat_len;
regex_cd->pattern_lens[regex_index] = pat_len;
regex_cd->patterns[regex_index] = ALLOC(char, pat_len);
memcpy(regex_cd->patterns[regex_index],
expr_array[i].patterns[j].pat,
expr_array[i].patterns[j].pat_len);
regex_index++;
}
exprs[i].items[j].item_id = pattern_id;
pattern_id++;
}
exprs[i].expr_id = expr_array[i].expr_id;
exprs[i].item_num = expr_array[i].n_patterns;
}
if (literal_cd != NULL) {
literal_cd->n_patterns = literal_index;
}
if (regex_cd != NULL) {
regex_cd->n_patterns = regex_index;
}
int ret = -1;
int max_patterns_type = 0;
size_t mem_size = 0;
struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1);
hs_instance->nr_worker_threads = nr_worker_threads;
hs_instance->n_patterns = pattern_id;
hs_instance->n_expr = n_expr_array;
hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
/* create bool matcher */
hs_instance->hs_rt->bm = bool_matcher_new(exprs, n_expr_array, &mem_size);
if (hs_instance->hs_rt->bm != NULL) {
/* log_info("Adapter_hs module: build bool matcher of %u expressions with %u bytes memory.",
n_expr_array, mem_size); */
} else {
/* log_error("Adapter_hs module: build bool matcher failed."); */
goto error;
}
free(exprs);
/* build hs database */
ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, scan_mode);
if (ret < 0) {
//log_error()
goto error;
}
if (literal_cd != NULL) {
adpt_hs_compile_data_free(literal_cd, literal_index);
}
if (regex_cd != NULL) {
adpt_hs_compile_data_free(regex_cd, regex_index);
}
/* which pattern type has more patterns, use it as hs_alloc_scratch's input parameter */
if (literal_pattern_num > regex_pattern_num) {
max_patterns_type = PATTERN_TYPE_STR;
} else {
max_patterns_type = PATTERN_TYPE_REG;
}
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, nr_worker_threads, max_patterns_type);
if (ret < 0) {
// log_error
goto error;
}
return hs_instance;
error:
adapter_hs_destroy(hs_instance);
return NULL;
}
void adapter_hs_destroy(struct adapter_hs *hs_instance)
{
if (NULL == hs_instance) {
return;
}
if (hs_instance->hs_rt != NULL) {
if (hs_instance->hs_rt->literal_db != NULL) {
hs_free_database(hs_instance->hs_rt->literal_db);
}
if (hs_instance->hs_rt->regex_db != NULL) {
hs_free_database(hs_instance->hs_rt->regex_db);
}
if (hs_instance->hs_rt->scratchs != NULL) {
for (size_t i = 0; i < hs_instance->nr_worker_threads; i++) {
if (hs_instance->hs_rt->scratchs[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->scratchs[i]);
}
}
}
free(hs_instance->hs_rt->scratchs);
if (hs_instance->hs_rt->bm != NULL) {
bool_matcher_free(hs_instance->hs_rt->bm);
}
free(hs_instance->hs_rt);
}
free(hs_instance);
}
static inline int compare_pattern_id(const void* a, const void* b)
{
long long ret= *(unsigned long long *)a - *(unsigned long long *)b;
if (0 == ret) {
return 0;
} else if (ret < 0) {
return -1;
} else {
return 1;
}
}
UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
/**
* @param id: pattern id
*/
int matched_event_cb(unsigned int id, unsigned long long from,
unsigned long long to, unsigned int flags, void *ctx) {
// put id in set
UT_array *pattern_id_set = (UT_array *)ctx;
unsigned long long pattern_id = (unsigned long long)id;
if (utarray_find(pattern_id_set, &pattern_id, compare_pattern_id)) {
return -1;
}
utarray_push_back(pattern_id_set, &pattern_id);
utarray_sort(pattern_id_set, compare_pattern_id);
return 0;
}
int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *data, size_t data_len,
int results[], size_t *n_results)
{
struct adapter_hs_runtime *hs_rt = hs_instance->hs_rt;
hs_scratch_t *scratch = hs_rt->scratchs[thread_id];
UT_array *pattern_id_set;
hs_error_t err;
utarray_new(pattern_id_set, &ut_pattern_id_icd);
utarray_reserve(pattern_id_set, hs_instance->n_patterns);
if (hs_rt->literal_db != NULL) {
err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set);
if (err != HS_SUCCESS) {
//log_error()
return -1;
}
}
if (hs_rt->regex_db != NULL) {
err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set);
if (err != HS_SUCCESS) {
//log_error()
return -1;
}
}
size_t pattern_set_size = utarray_len(pattern_id_set);
unsigned long long items[pattern_set_size];
memset(items, 0, sizeof(unsigned long long) * pattern_set_size);
for (size_t i = 0; i < pattern_set_size; i++) {
items[i] = *(unsigned long long *)utarray_eltptr(pattern_id_set, i);
}
size_t matched_index = 0;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr);
size_t bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_instance->n_expr);
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
results[matched_index] = bool_matcher_results[matched_index].expr_id;
}
*n_results = bool_matcher_ret;
free(bool_matcher_results);
utarray_free(pattern_id_set);
return 0;
}
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id)
{
struct adapter_hs_stream *hs_stream = ALLOC(struct adapter_hs_stream, 1);
hs_error_t err;
hs_stream->thread_id = thread_id;
hs_stream->n_expr = hs_instance->n_expr;
hs_stream->n_patterns = hs_instance->n_patterns;
hs_stream->hs_rt = hs_instance->hs_rt;
utarray_new(hs_stream->pattern_id_set, &ut_pattern_id_icd);
utarray_reserve(hs_stream->pattern_id_set, hs_stream->n_patterns);
if (hs_instance->hs_rt->literal_db != NULL) {
err = hs_open_stream(hs_instance->hs_rt->literal_db, 0, &hs_stream->literal_stream);
if (err != HS_SUCCESS) {
// log_error
return NULL;
}
}
if (hs_instance->hs_rt->regex_db != NULL) {
err = hs_open_stream(hs_instance->hs_rt->regex_db, 0, &hs_stream->regex_stream);
if (err != HS_SUCCESS) {
// log_error
return NULL;
}
}
return hs_stream;
}
int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len,
int results[], size_t *n_results)
{
hs_error_t err;
int thread_id = hs_stream->thread_id;
if (hs_stream->literal_stream != NULL) {
err = hs_scan_stream(hs_stream->literal_stream, data, data_len, 0, hs_stream->hs_rt->scratchs[thread_id],
matched_event_cb, hs_stream->pattern_id_set);
if (err != HS_SUCCESS) {
//log_error()
return -1;
}
}
if (hs_stream->regex_stream != NULL) {
err = hs_scan_stream(hs_stream->regex_stream, data, data_len, 0, hs_stream->hs_rt->scratchs[thread_id],
matched_event_cb, hs_stream->pattern_id_set);
if (err != HS_SUCCESS) {
//log_error()
return -1;
}
}
size_t pattern_set_size = utarray_len(hs_stream->pattern_id_set);
unsigned long long items[pattern_set_size];
memset(items, 0, sizeof(unsigned long long) * pattern_set_size);
for (size_t i = 0; i < pattern_set_size; i++) {
items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->pattern_id_set, i);
}
size_t matched_index = 0;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_stream->n_expr);
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
results[matched_index] = bool_matcher_results[matched_index].expr_id;
}
*n_results = bool_matcher_ret;
free(bool_matcher_results);
return 0;
}
void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
{
int thread_id = hs_stream->thread_id;
hs_close_stream(hs_stream->literal_stream, hs_stream->hs_rt->scratchs[thread_id], NULL, NULL);
hs_close_stream(hs_stream->regex_stream, hs_stream->hs_rt->scratchs[thread_id], NULL, NULL);
utarray_free(hs_stream->pattern_id_set);
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */
hs_stream->hs_rt = NULL;
free(hs_stream);
}

216
src/bool_matcher.cpp
View File

@@ -1,216 +0,0 @@
#include "bool_matcher.h"
#include <map>
#include <vector>
#include <algorithm>
using namespace std;
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
struct bool_expr_item
{
size_t item_num;
struct bool_item * items;
};
struct bool_matcher
{
unsigned int bool_expr_num;
struct bool_expr_match * bool_expr_ids;
struct bool_expr_item * bool_expr_items;
unsigned int bool_item_num;
unsigned long long * bool_items;
unsigned int * mapped_ptr;
unsigned int * mapped_ids;
unsigned int bitmap_size;
unsigned char * bitmap;
};
bool operator<(const struct bool_item & lhs, const struct bool_item & rhs)
{
return lhs.item_id<rhs.item_id;
}
struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, size_t * mem_size)
{
if(exprs==NULL || expr_num==0) return NULL;
unsigned int mem_bytes=0;
struct bool_matcher * matcher=new struct bool_matcher;
mem_bytes+=sizeof(bool_matcher);
matcher->bool_expr_num=(unsigned int)expr_num;
matcher->bool_expr_ids =new struct bool_expr_match[expr_num];
matcher->bool_expr_items=new struct bool_expr_item[expr_num];
mem_bytes+=(unsigned int)expr_num*(sizeof(struct bool_expr_match)+sizeof(struct bool_expr_item));
for(unsigned int i=0; i<expr_num; i++)
{
matcher->bool_expr_ids[i].expr_id =exprs[i].expr_id;
matcher->bool_expr_ids[i].user_tag =exprs[i].user_tag;
matcher->bool_expr_items[i].item_num=exprs[i].item_num;
matcher->bool_expr_items[i].items=new struct bool_item[exprs[i].item_num];
mem_bytes+=(unsigned int)exprs[i].item_num*sizeof(struct bool_item);
copy(exprs[i].items, exprs[i].items+exprs[i].item_num, matcher->bool_expr_items[i].items);
sort(matcher->bool_expr_items[i].items, matcher->bool_expr_items[i].items+exprs[i].item_num);
}
map<unsigned long long, unsigned int> M1;
for(unsigned int i=0; i<expr_num; i++)
{
for(unsigned int j=0; j<exprs[i].item_num; j++)
{
if(exprs[i].items[j].not_flag==0) M1[exprs[i].items[j].item_id]++;
}
}
map< unsigned long long, vector<unsigned int> > M2;
for(unsigned int i=0; i<expr_num; i++)
{
unsigned int min_count=-1;
unsigned long long item_id;
for(unsigned int j=0; j<exprs[i].item_num; j++)
{
if(exprs[i].items[j].not_flag==0)
{
unsigned int c=M1[exprs[i].items[j].item_id];
if(c<min_count)
{
min_count=c;
item_id=exprs[i].items[j].item_id;
}
}
}
M2[item_id].push_back(i);
}
matcher->bool_item_num=(unsigned int)M2.size();
matcher->bool_items =new unsigned long long[M2.size()];
matcher->mapped_ptr =new unsigned int[M2.size()+1];
matcher->mapped_ids =new unsigned int[matcher->bool_expr_num];
mem_bytes+=((unsigned int)M2.size()+1+matcher->bool_expr_num)*sizeof(unsigned int)+(unsigned int)M2.size()*sizeof(unsigned long long);
matcher->mapped_ptr[0]=0;
map< unsigned long long, vector<unsigned int> >::const_iterator it=M2.begin();
for(unsigned int k=0; k<M2.size(); ++k, ++it)
{
matcher->bool_items[k]=it->first;
copy(it->second.begin(), it->second.end(), matcher->mapped_ids+matcher->mapped_ptr[k]);
matcher->mapped_ptr[k+1]=matcher->mapped_ptr[k]+(unsigned int)it->second.size();
}
M1.clear();
M2.clear();
matcher->bitmap_size=(1U<<27);
matcher->bitmap=new unsigned char[(matcher->bitmap_size)>>3];
mem_bytes+=(matcher->bitmap_size)>>3;
memset(matcher->bitmap, 0, (matcher->bitmap_size)>>3);
for(unsigned int i=0; i<matcher->bool_item_num; i++)
{
unsigned int j=matcher->bool_items[i]&(matcher->bitmap_size-1);
matcher->bitmap[j>>3]|=(1U<<(j&7));
}
if(mem_size!=NULL) *mem_size=mem_bytes;
return matcher;
}
int res_comp(const void * lhs, const void * rhs)
{
bool_expr_match * _lhs=(bool_expr_match *)lhs;
bool_expr_match * _rhs=(bool_expr_match *)rhs;
return (_lhs->expr_id<_rhs->expr_id) ? 1 : -1;
}
int do_match(struct bool_expr_item * expr, unsigned long long * item_ids, size_t item_num)
{
unsigned int i=0;
for(unsigned int j=0; j<expr->item_num; ++j)
{
if(expr->items[j].not_flag==0)
{
while(i<item_num && item_ids[i]<expr->items[j].item_id) ++i;
if(i==item_num || item_ids[i]>expr->items[j].item_id) return 0;
++i;
}
else
{
while(i<item_num && item_ids[i]<expr->items[j].item_id) ++i;
if(i<item_num && item_ids[i]==expr->items[j].item_id) return 0;
}
}
return 1;
}
int bool_matcher_match(struct bool_matcher * matcher, unsigned long long * item_ids, size_t item_num, struct bool_expr_match * results, size_t n_result)
{
if(matcher==NULL) return -1;
if(item_num==0) return 0;
// sort(item_ids, item_ids+item_num);
// size_t J=0;
// for(unsigned int i=1; i<item_num; i++)
// {
// if(item_ids[i]!=item_ids[J]) item_ids[++J]=item_ids[i];
// }
// item_num=J+1;
unsigned int r=0;
for(unsigned int i=0; i<item_num; i++)
{
unsigned int t=item_ids[i]&(matcher->bitmap_size-1);
if((matcher->bitmap[t>>3]&(1U<<(t&7)))==0) continue;
int l=0, h=(int)matcher->bool_item_num-1;
while(l<=h)
{
int m=(l+h)/2;
if(item_ids[i]==matcher->bool_items[m])
{
for(unsigned int j=matcher->mapped_ptr[m]; j<matcher->mapped_ptr[m+1]; j++)
{
unsigned int idx=matcher->mapped_ids[j];
int ret=do_match(matcher->bool_expr_items+idx, item_ids, item_num);
if(ret==1)
{
if(r==n_result) goto END;
results[r++]=matcher->bool_expr_ids[idx];
}
}
break;
}
else if(item_ids[i]<matcher->bool_items[m])
{
h=m-1;
}
else
{
l=m+1;
}
}
}
END:
qsort(results, r, sizeof(bool_expr_match), res_comp);
return r;
}
void bool_matcher_free(struct bool_matcher * matcher)
{
if(matcher==NULL) return;
delete [] matcher->bool_expr_ids;
for(unsigned int i=0; i<matcher->bool_expr_num; i++) delete [] matcher->bool_expr_items[i].items;
delete [] matcher->bool_expr_items;
delete [] matcher->bool_items;
delete [] matcher->mapped_ptr;
delete [] matcher->mapped_ids;
delete [] matcher->bitmap;
delete matcher;
return;
}

2932
src/cJSON.c
View File

File diff suppressed because it is too large Load Diff

103
src/inc_internal/adapter_hs.h
View File

@@ -1,103 +0,0 @@
/*
**********************************************************************************************
* File: adapter_hs.h
* Description: wrapper for raw hyperscan
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _ADAPTER_HS_H_
#define _ADAPTER_HS_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include <stddef.h>
#include <stdint.h>
#define MAX_EXPR_PATTERN_NUM 8
struct adapter_hs;
/* scan mode */
enum {
SCAN_MODE_BLOCK = 1,
SCAN_MODE_STREAM,
};
/* pattern type: PATTERN_TYPE_STR(pure literal string) or PATTERN_TYPE_REG(regex expression) */
enum {
PATTERN_TYPE_STR = 1,
PATTERN_TYPE_REG,
};
typedef struct {
/* pattern type */
int type;
/* start pointer of pattern */
char *pat;
/* pattern length */
size_t pat_len;
} scan_pattern_t;
/* logic AND expression, such as (pattern1 & pattern2) */
typedef struct {
uint32_t expr_id;
size_t n_patterns;
scan_pattern_t patterns[MAX_EXPR_PATTERN_NUM];
} and_expr_t;
/**
* @brief initialize adapter_hs instance
*
* @param scan_mode: the following scan as block or stream mode
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
* @param expr_array: logic AND expression's array
* @param n_expr_arrays: the number of logic AND expression's array
*
* @retval the pointer to adapter_hs instance
*/
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id
*
* @param instance: adapter_hs instance obtained by adapter_hs_initialize()
* @param thread_id: the thread_id of caller
* @param data: data to be scanned
* @param data_len: the length of data to be scanned
* @param results: the array of expr_id
* @param n_results: number of elements in array of expr_id
*/
int adapter_hs_scan(struct adapter_hs *instance, int thread_id, const char *data, size_t data_len,
int results[], size_t *n_results);
/**
* @brief destroy adapter_hs instance
*
* @param instance: adapter_hs instance obtained by adapter_hs_initialize()
*/
void adapter_hs_destroy(struct adapter_hs *instance);
struct adapter_hs_stream;
/**
* @brief open adapter_hs stream after adapter_hs instance initialized for stream scan
*
*/
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id);
int adapter_hs_scan_stream(struct adapter_hs_stream *stream, const char *data, size_t data_len,
int results[], size_t *n_results);
void adapter_hs_stream_close(struct adapter_hs_stream *stream);
#ifdef __cpluscplus
}
#endif
#endif

66
src/inc_internal/bool_matcher.h
View File

@@ -1,66 +0,0 @@
/*
*
* Copyright (c) 2018
* String Algorithms Research Group
* Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
* National Engineering Laboratory for Information Security Technologies (NELIST)
* All rights reserved
*
* Written by: LIU YANBING (liuyanbing@iie.ac.cn)
* Last modification: 2021-06-12
*
* This code is the exclusive and proprietary property of IIE-CAS and NELIST.
* Usage for direct or indirect commercial advantage is not allowed without
* written permission from the authors.
*
*/
#ifndef INCLUDE_BOOL_MATCHER_H
#define INCLUDE_BOOL_MATCHER_H
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#define MAX_ITEMS_PER_BOOL_EXPR 8
/* not_flag=0表示布尔项item_id必须出现；not_flag=1表示布尔项item_id不能出现 */
struct bool_item
{
unsigned long long item_id;
unsigned char not_flag;
};
/* At least one item's not_flag should be 0. */
struct bool_expr
{
unsigned long long expr_id;
void *user_tag;
size_t item_num;
struct bool_item items[MAX_ITEMS_PER_BOOL_EXPR];
};
struct bool_expr_match
{
unsigned long long expr_id;
void *user_tag;
};
struct bool_matcher;
struct bool_matcher *bool_matcher_new(struct bool_expr *exprs, size_t expr_num, size_t *mem_size);
/* Returned results are sorted by expr_id in descending order. */
// Input item_ids MUST be ASCENDING order and NO duplication.
int bool_matcher_match(struct bool_matcher *matcher, unsigned long long *item_ids, size_t item_num, struct bool_expr_match *results, size_t n_result);
void bool_matcher_free(struct bool_matcher *matcher);
#ifdef __cplusplus
}
#endif
#endif

277
src/inc_internal/cJSON.h
View File

@@ -1,277 +0,0 @@
/*
Copyright (c) 2009-2017 Dave Gamble and cJSON contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef cJSON__h
#define cJSON__h
#ifdef __cplusplus
extern "C"
{
#endif
/* project version */
#define CJSON_VERSION_MAJOR 1
#define CJSON_VERSION_MINOR 7
#define CJSON_VERSION_PATCH 7
#include <stddef.h>
/* cJSON Types: */
#define cJSON_Invalid (0)
#define cJSON_False (1 << 0)
#define cJSON_True (1 << 1)
#define cJSON_NULL (1 << 2)
#define cJSON_Number (1 << 3)
#define cJSON_String (1 << 4)
#define cJSON_Array (1 << 5)
#define cJSON_Object (1 << 6)
#define cJSON_Raw (1 << 7) /* raw json */
#define cJSON_IsReference 256
#define cJSON_StringIsConst 512
/* The cJSON structure: */
typedef struct cJSON
{
/* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */
struct cJSON *next;
struct cJSON *prev;
/* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */
struct cJSON *child;
/* The type of the item, as above. */
int type;
/* The item's string, if type==cJSON_String and type == cJSON_Raw */
char *valuestring;
/* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */
int valueint;
/* The item's number, if type==cJSON_Number */
double valuedouble;
/* The item's name string, if this item is the child of, or is in the list of subitems of an object. */
char *string;
} cJSON;
typedef struct cJSON_Hooks
{
void *(*malloc_fn)(size_t sz);
void (*free_fn)(void *ptr);
} cJSON_Hooks;
typedef int cJSON_bool;
#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32))
#define __WINDOWS__
#endif
#ifdef __WINDOWS__
/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 2 define options:
CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols
CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default)
CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol
For *nix builds that support visibility attribute, you can define similar behavior by
setting default visibility to hidden by adding
-fvisibility=hidden (for gcc)
or
-xldscope=hidden (for sun cc)
to CFLAGS
then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does
*/
/* export symbols by default, this is necessary for copy pasting the C and header file */
#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS)
#define CJSON_EXPORT_SYMBOLS
#endif
#if defined(CJSON_HIDE_SYMBOLS)
#define CJSON_PUBLIC(type) type __stdcall
#elif defined(CJSON_EXPORT_SYMBOLS)
#define CJSON_PUBLIC(type) __declspec(dllexport) type __stdcall
#elif defined(CJSON_IMPORT_SYMBOLS)
#define CJSON_PUBLIC(type) __declspec(dllimport) type __stdcall
#endif
#else /* !WIN32 */
#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY)
#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type
#else
#define CJSON_PUBLIC(type) type
#endif
#endif
/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them.
* This is to prevent stack overflows. */
#ifndef CJSON_NESTING_LIMIT
#define CJSON_NESTING_LIMIT 1000
#endif
/* returns the version of cJSON as a string */
CJSON_PUBLIC(const char*) cJSON_Version(void);
/* Supply malloc, realloc and free functions to cJSON */
CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks);
/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */
/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */
CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value);
/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */
/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */
CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated);
/* Render a cJSON entity to text for transfer/storage. */
CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item);
/* Render a cJSON entity to text for transfer/storage without any formatting. */
CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item);
/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */
CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt);
/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */
/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */
CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format);
/* Delete a cJSON entity and all subentities. */
CJSON_PUBLIC(void) cJSON_Delete(cJSON *c);
/* Returns the number of items in an array (or object). */
CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array);
/* Retrieve item number "index" from array "array". Returns NULL if unsuccessful. */
CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index);
/* Get item "string" from object. Case insensitive. */
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string);
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string);
CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string);
/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */
CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void);
/* Check if the item is a string and return its valuestring */
CJSON_PUBLIC(char *) cJSON_GetStringValue(cJSON *item);
/* These functions check the type of an item */
CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item);
CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item);
/* These calls create a cJSON item of the appropriate type. */
CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean);
CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num);
CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string);
/* raw json */
CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw);
CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void);
CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void);
/* Create a string where valuestring references a string so
* it will not be freed by cJSON_Delete */
CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string);
/* Create an object/arrray that only references it's elements so
* they will not be freed by cJSON_Delete */
CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child);
CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child);
/* These utilities create an Array of count items. */
CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count);
CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count);
/* Append item to the specified array/object. */
CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item);
CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item);
/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object.
* WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before
* writing to `item->string` */
CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item);
/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */
CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item);
CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item);
/* Remove/Detatch items from Arrays/Objects. */
CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which);
CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string);
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string);
CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string);
CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string);
/* Update array items. */
CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement);
CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem);
CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem);
CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem);
/* Duplicate a cJSON item */
CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse);
/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will
need to be released. With recurse!=0, it will duplicate any children connected to the item.
The item->next and ->prev pointers are always zero on return from Duplicate. */
/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal.
* case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */
CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive);
CJSON_PUBLIC(void) cJSON_Minify(char *json);
/* Helper functions for creating and adding items to an object at the same time.
* They return the added item or NULL on failure. */
CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean);
CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number);
CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string);
CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw);
CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name);
CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name);
/* When assigning an integer value, it needs to be propagated to valuedouble too. */
#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number))
/* helper for the cJSON_SetNumberValue macro */
CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number);
#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number))
/* Macro for iterating over an array or object */
#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next)
/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */
CJSON_PUBLIC(void *) cJSON_malloc(size_t size);
CJSON_PUBLIC(void) cJSON_free(void *object);
#ifdef __cplusplus
}
#endif
#endif

7
src/inc_internal/maat_common.h
View File

@@ -20,6 +20,13 @@ extern "C"
struct maat_options {
size_t nr_worker_threads;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
int gc_timeout_ms;
int deferred_load_on;
enum data_source input_mode;
char iris_full_dir[NAME_MAX];
char iris_inc_dir[NAME_MAX];
};
#ifdef __cpluscplus

13
src/inc_internal/maat_config_monitor.h
View File

@@ -18,15 +18,14 @@ extern "C"
#include <stdint.h>
#define CONFIG_UPDATE_TYPE_NONE 0
#define CONFIG_UPDATE_TYPE_FULL 1
#define CONFIG_UPDATE_TYPE_INC 2
#define CM_UPDATE_TYPE_NONE 0
#define CM_UPDATE_TYPE_FULL 1
#define CM_UPDATE_TYPE_INC 2
void config_monitor_traverse(uint64_t version, const char *idx_dir,
void (*pre_fn)(uint64_t, int, void *),
void config_monitor_traverse(long long version, const char *idx_dir,
void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *),
void (*post_fn)(void *),
void (*finish_fn)(void *),
void *u_param);
#ifdef __cpluscplus

37
src/inc_internal/maat_ex_data.h
View File

@@ -16,12 +16,47 @@ extern "C"
{
#endif
#include "rcu_hash.h"
struct ex_data_runtime;
struct ex_data_runtime *ex_data_runtime_new(void (* data_free)(void *data));
/* ex_data_runtime API */
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn);
void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt);
void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt);
/* ex_data_runtime cache row API */
void ex_data_runtime_cache_row_put(struct ex_data_runtime *ex_data_rt, const char *row);
const char *ex_data_runtime_cached_row_get(struct ex_data_runtime *ex_data_rt, size_t index);
size_t ex_data_runtime_cached_row_count(struct ex_data_runtime *ex_data_rt);
void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt);
/* set schema API */
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema);
/* set user_ctx API */
void ex_data_runtime_set_user_ctx(struct ex_data_runtime *ex_data_rt, void *user_ctx);
/* ex_data_runtime ex data API */
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len);
void ex_data_runtime_add_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, void *data);
void ex_data_runtime_del_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
size_t ex_data_runtime_ex_data_count(struct ex_data_runtime *ex_data_rt);
size_t ex_data_runtime_list_updating_ex_data(struct ex_data_runtime *ex_data_rt, void ***ex_data_array);
int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt);
#ifdef __cpluscplus
}
#endif

4
src/inc_internal/maat_kv.h
View File

@@ -8,8 +8,8 @@
***********************************************************************************************
*/
#ifndef _MAAT_KV_MAP_H_
#define _MAAT_KV_MAP_H_
#ifndef _MAAT_KV_H_
#define _MAAT_KV_H_
#ifdef __cpluscplus
extern "C"

48
src/inc_internal/maat_rule.h
View File

@@ -24,49 +24,67 @@ extern "C"
struct maat_runtime {
/* maat_runtime can be created and destroy dynamic, so need version info */
uint64_t version;
long long version;
time_t last_update_time;
struct maat_table_runtime_manager *table_rt_mgr;
struct table_runtime_manager *table_rt_mgr;
size_t max_table_num;
int max_thread_num;
size_t max_thread_num;
uint32_t rule_num;
};
enum rule_import_type {
RULE_IMPORT_TYPE_IRIS = 1,
RULE_IMPORT_TYPE_MAX
enum data_source {
DATA_SOURCE_NONE = 0,
DATA_SOURCE_IRIS_FILE
};
struct rule_import_iris_ctx {
struct source_iris_ctx {
char inc_dir[NAME_MAX];
char full_dir[NAME_MAX];
};
struct maat {
struct maat_runtime *maat_rt;
struct maat_runtime *rebuilding_maat_rt; //TODO: creating
//struct maat_garbage_collector *gc;
struct maat_table_manager *table_mgr;
char instance_name[NAME_MAX];
enum rule_import_type rule_import_type;
struct maat_runtime *maat_rt;
struct maat_runtime *creating_maat_rt;
struct table_schema_manager *table_schema_mgr;
enum data_source input_mode;
union {
struct rule_import_iris_ctx iris_ctx;
struct source_iris_ctx iris_ctx;
};
int deferred_load;
int is_running;
pthread_mutex_t background_update_mutex;
int nr_worker_thread;
uint64_t maat_version;
uint64_t last_full_version;
long long maat_version;
long long last_full_version;
pthread_t cfg_mon_thread;
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
int gc_timeout_ms; //garbage collection timeout_ms;
struct maat_garbage_bin *garbage_bin;
};
void maat_start_cb(long long new_version, int update_type, void *u_para);
int maat_update_cb(const char *table_name, const char *line, void *u_para);
void maat_finish_cb(void *u_para);
void *rule_monitor_loop(void *arg);
void maat_read_full_config(struct maat *maat_instance);
#ifdef __cpluscplus
}
#endif

48
src/inc_internal/maat_table_runtime.h
View File

@@ -19,19 +19,51 @@ extern "C"
#include "maat_table_schema.h"
#include "maat_garbage_collection.h"
struct maat_table_item;
struct maat_table_runtime;
struct maat_table_runtime_manager;
struct table_item;
struct table_runtime;
struct table_runtime_manager;
struct maat_table_runtime_manager *maat_table_runtime_manager_create(struct maat_table_manager *table_mgr, int max_thread_num, struct maat_garbage_bin* bin);
/* table runtime manager API */
struct table_runtime_manager *table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num,
struct maat_garbage_bin *bin);
void maat_table_runtime_manager_destroy(struct maat_table_runtime_manager *table_rt_mgr);
void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr);
struct maat_table_runtime *maat_table_runtime_get(struct maat_table_runtime_manager *table_rt_mgr, int table_id);
/* table runtime API */
struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id);
enum maat_table_type maat_table_runtime_get_type(struct maat_table_runtime* table_rt);
size_t table_runtime_rule_count(struct table_runtime *table_rt);
void maat_table_runtime_item_add(struct maat_table_runtime *table_rt, struct maat_table_item *table_item);
enum table_type table_runtime_get_type(struct table_runtime* table_rt);
void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, const char *line, struct table_item *table_item);
/**
* @brief if table_runtime is updating
*
* @retval 1(yes) 0(no)
*/
int table_runtime_updating_flag(struct table_runtime *table_rt);
void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread);
/* table runtime scan API */
int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len,
int results[], size_t *n_result);
void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id);
int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len,
int result[], size_t n_result);
void table_runtime_stream_close(struct table_runtime *table_rt);
/* plugin table runtime API */
size_t plugin_table_runtime_cached_row_count(struct table_runtime *table_rt);
const char* plugin_table_runtime_get_cached_row(struct table_runtime *table_rt, size_t row_seq);
void *plugin_table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema, const char *key, size_t key_len);
void plugin_table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema);
#ifdef __cpluscplus
}

147
src/inc_internal/maat_table_schema.h
View File

@@ -18,27 +18,152 @@ extern "C"
#include <stddef.h>
#include "sds/sds.h"
#include "maat/maat.h"
#include "adapter_hs.h"
enum maat_table_type {
#define MAX_DISTRICT_STR 128
#define MAX_IP_STR 128
#define MAX_KEYWORDS_STR 1024
enum table_type {
TABLE_TYPE_EXPR = 0,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_IP,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_MAX
};
struct maat_table_schema;
struct maat_table_manager;
struct maat_table_item;
enum expr_type {
EXPR_TYPE_STRING = 1,
EXPR_TYPE_REGEX,
EXPR_TYPE_AND,
EXPR_TYPE_OFFSET,
EXPR_TYPE_MAX
};
struct maat_table_manager *maat_table_manager_create(sds table_info_path);
void maat_table_manager_destroy(struct maat_table_manager *table_mgr);
enum match_method {
MATCH_METHOD_SUB=0,
MATCH_METHOD_RIGHT,
MATCH_METHOD_LEFT,
MATCH_METHOD_COMPLETE,
MATCH_METHOD_MAX
};
int maat_table_manager_get_table_id(struct maat_table_manager* table_mgr, sds table_name);
enum maat_table_type maat_table_manager_get_table_type(struct maat_table_manager *table_mgr, int id);
struct expr_item {
int item_id;
int group_id;
char district[MAX_DISTRICT_STR];
char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type;
enum match_method match_method;
int is_hexbin;
int is_case_sensitive;
int is_valid;
//rule_tag; 只存在schema里
//int have_exdata;
//struct ex_data *ex_data; //hash表
};
size_t maat_table_manager_get_size(struct maat_table_manager* table_mgr);
struct plugin_item {
char key[MAX_KEYWORDS_STR];
size_t key_len;
int is_valid;
};
struct maat_table_item *maat_table_line_to_item(sds line, struct maat_table_schema *table_schema);
struct ip_plugin_item {
int item_id;
int ip_type;
char start_ip[MAX_IP_STR];
char end_ip[MAX_IP_STR];
int is_valid;
int rule_tag;
int have_exdata;
void *ex_data;
};
struct table_item {
enum table_type table_type;
union {
struct expr_item expr_item;
struct plugin_item plugin_item;
struct ip_plugin_item ip_plugin_item;
};
};
struct plugin_table_callback_schema
{
maat_start_callback_t *start;
maat_update_callback_t *update;
maat_finish_callback_t *finish;
void* u_para;
};
struct ex_data_schema
{
maat_plugin_ex_new_func_t *new_func;
maat_plugin_ex_free_func_t *free_func;
maat_plugin_ex_dup_func_t *dup_func;
//Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
int set_flag;
};
struct table_schema;
struct table_schema_manager;
/* table schema manager API */
struct table_schema_manager *table_schema_manager_create(const char *table_info_path);
void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr);
int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name);
enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id);
size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_mgr);
void table_schema_manager_all_plugin_cb_start(struct table_schema_manager* table_schema_mgr, int update_type);
void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* table_schema_mgr);
/* table schema generic API */
struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id);
enum table_type table_schema_get_table_type(struct table_schema *table_schema);
int table_schema_get_table_id(struct table_schema *table_schema);
struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema);
/* expr table schema API */
enum scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);
/* plugin table schema API */
int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp);
struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schema *table_schema);
/**
* @brief if plugin table schema's ex data schema set
*
* @retval 1(already Set) 0(Not set yet)
*/
int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema);
int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_mgr, int table_id,
maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
maat_update_callback_t *update,//table line ,u_para
maat_finish_callback_t *finish,//u_para
void *u_para);
/**
* @brief the number of callback function stored in plugin table schema
*/
size_t plugin_table_schema_callback_count(struct table_schema *table_schema);
void plugin_table_schema_all_cb_update(struct table_schema *table_schema, const char *row);
#ifdef __cpluscplus
}

23
src/inc_internal/maat_utils.h
View File

@@ -19,11 +19,28 @@ extern "C"
#include <stdlib.h>
#include <stddef.h>
#include "sds/sds.h"
#define TRUE 1
#define FALSE 0
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
int get_column_pos(sds line, int column_seq, size_t *offset, size_t *len);
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
char *maat_strdup(const char *s);
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len);
int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *out_sz);
char *strtok_r_esc(char *s, const char delim, char **save_ptr);
char *str_unescape_and(char *s);
char *str_unescape(char *s);
#ifdef __cpluscplus
}

24
src/inc_internal/rcu_hash.h
View File

@@ -18,16 +18,20 @@ extern "C"
#include "uthash/uthash.h"
typedef void rcu_hash_data_free_fn(void *user_ctx, void *data);
/* rcu hash table */
struct rcu_hash_table;
struct rcu_hash_table *rcu_hash_new(void (* data_free)(void *data));
struct rcu_hash_table *rcu_hash_new(rcu_hash_data_free_fn *free_fn);
void rcu_hash_free(struct rcu_hash_table *htable);
void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx);
/**
* @brief the data added just in updating stage
* after call rcu_hash_commit, it in effective stage
* @brief just means add to the updating nodes
* after call rcu_hash_commit, they become effective nodes
*/
void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len, void *data);
@@ -44,14 +48,24 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len
*/
void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len);
size_t rcu_hash_counts(struct rcu_hash_table *htable);
size_t rcu_hash_count(struct rcu_hash_table *htable);
/**
* @brief make add/del effective
*/
void rcu_hash_commit(struct rcu_hash_table *htable);
size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable);
size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array);
/**
* @brief check if rcu hash table is updating
*
* @retval 1 means htable is updating, otherwise 0
*/
int rcu_hash_updating_flag(struct rcu_hash_table *htable);
#ifdef __cpluscplus
}
#endif

229
src/maat_api.cpp
View File

@@ -9,19 +9,27 @@
*/
#include <stdio.h>
#include <string.h>
#include "maat_utils.h"
#include "utils.h"
#include "maat/maat.h"
#include "maat_rule.h"
#include "maat_common.h"
#include "maat_kv.h"
#include "maat_table_schema.h"
#include "maat_table_runtime.h"
#include "sds/sds.h"
#include "maat_config_monitor.h"
struct maat_options* maat_options_new(void)
{
struct maat_options *options = ALLOC(struct maat_options, 1);
options->nr_worker_threads = 1;
options->deferred_load_on = 0;
options->rule_effect_interval_ms = 60 * 1000;
options->rule_update_checking_interval_ms = 1 * 1000;
options->gc_timeout_ms = 10 * 1000;
options->input_mode = DATA_SOURCE_NONE;
return options;
}
@@ -33,25 +41,129 @@ int maat_options_set_worker_thread_number(struct maat_options *opts, size_t n_wo
return 0;
}
struct maat *maat_new(struct maat_options opts, const char *table_info_path)
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms)
{
opts->rule_effect_interval_ms = interval_ms;
return 0;
}
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms)
{
opts->rule_update_checking_interval_ms = interval_ms;
return 0;
}
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms)
{
opts->gc_timeout_ms = interval_ms;
return 0;
}
int maat_options_set_deferred_load_on(struct maat_options *opts)
{
opts->deferred_load_on = 1;
return 0;
}
int maat_options_set_iris_full_dir(struct maat_options *opts, const char *full_dir)
{
if (strlen(full_dir) >= NAME_MAX) {
return -1;
}
memcpy(opts->iris_full_dir, full_dir, strlen(full_dir));
opts->input_mode = DATA_SOURCE_IRIS_FILE;
return 0;
}
int maat_options_set_iris_inc_dir(struct maat_options *opts, const char *inc_dir)
{
if (strlen(inc_dir) >= NAME_MAX) {
return -1;
}
memcpy(opts->iris_inc_dir, inc_dir, strlen(inc_dir));
opts->input_mode = DATA_SOURCE_IRIS_FILE;
return 0;
}
void maat_read_full_config(struct maat *maat_instance)
{
switch (maat_instance->input_mode) {
case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version,
maat_instance->iris_ctx.full_dir,
maat_start_cb,
maat_update_cb,
maat_finish_cb,
maat_instance);
if (NULL == maat_instance->creating_maat_rt) {
fprintf(stderr, "At initiation: NO effective rule in %s.",
maat_instance->iris_ctx.full_dir);
}
break;
default:
break;
}
maat_instance->maat_rt = maat_instance->creating_maat_rt;
maat_instance->creating_maat_rt = NULL;
maat_instance->is_running = 1;
if (maat_instance->maat_rt != NULL) {
maat_instance->maat_version = maat_instance->maat_rt->version;
maat_instance->last_full_version = maat_instance->maat_rt->version;
}
}
struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
{
#if 0
if (NULL == table_info_path) {
return NULL;
}
int garbage_gc_timeout_s = 0;
struct maat *maat_instance = ALLOC(struct maat, 1);
maat_instance->table_mgr = maat_table_manager_create(table_info_path);
maat_instance->table_schema_mgr = table_schema_manager_create(table_info_path);
if (NULL == maat_instance->table_schema_mgr) {
goto failed;
}
maat_instance->input_mode = opts->input_mode;
switch (maat_instance->input_mode) {
case DATA_SOURCE_IRIS_FILE:
memcpy(maat_instance->iris_ctx.full_dir, opts->iris_full_dir, strlen(opts->iris_full_dir));
memcpy(maat_instance->iris_ctx.inc_dir, opts->iris_inc_dir, strlen(opts->iris_inc_dir));
break;
default:
fprintf(stderr, "data source unsupported:%d\n", maat_instance->input_mode);
goto failed;
}
maat_instance->is_running = 0;
maat_instance->maat_version = 0;
maat_instance->last_full_version = 0;
maat_instance->nr_worker_thread = opts.nr_worker_threads;
maat_instance->nr_worker_thread = opts->nr_worker_threads;
maat_instance->rule_effect_interval_ms = opts->rule_effect_interval_ms;
maat_instance->gc_timeout_ms = opts->gc_timeout_ms;
maat_instance->deferred_load = opts->deferred_load_on;
garbage_gc_timeout_s = (maat_instance->rule_effect_interval_ms / 1000) +
(maat_instance->gc_timeout_ms / 1000);
maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s);
pthread_mutex_init(&(maat_instance->background_update_mutex), NULL);
if (0 == maat_instance->deferred_load) {
maat_read_full_config(maat_instance);
}
pthread_create(&(maat_instance->cfg_mon_thread), NULL, rule_monitor_loop, (void*)maat_instance);
return maat_instance;
#endif
failed:
free(maat_instance);
return NULL;
}
@@ -64,11 +176,100 @@ void maat_free(struct maat *maat_instance)
return;
}
int maat_table_get_id(struct maat *instance, const char *table_name)
int maat_table_get_id(struct maat *maat_instance, const char *table_name)
{
int table_id = -1;
struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr;
table_id = table_schema_manager_get_table_id(table_schema_mgr, table_name);
return table_id;
}
int maat_table_callback_register(struct maat *maat_instance, int table_id,
maat_start_callback_t *start,
maat_update_callback_t *update,
maat_finish_callback_t *finish,
void *u_para)
{
int ret = -1;
pthread_mutex_lock(&(maat_instance->background_update_mutex));
ret = plugin_table_schema_add_callback(maat_instance->table_schema_mgr, table_id, start, update, finish, u_para);
if (ret < 0) {
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return -1;
}
if (!maat_instance->maat_rt) {
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return 0;
}
struct table_runtime *table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id);
size_t row_count = plugin_table_runtime_cached_row_count(table_rt);
if (row_count > 0) {
if (start != NULL) {
start(MAAT_RULE_UPDATE_TYPE_FULL, u_para);
}
for (size_t i = 0; i < row_count; i++) {
const char *line = plugin_table_runtime_get_cached_row(table_rt, i);
if (NULL == line) {
break;
}
update(table_id, line, u_para);
}
if (finish != NULL) {
finish(u_para);
}
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return 0;
}
int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_id,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp)
{
struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
pthread_mutex_lock(&(maat_instance->background_update_mutex));
int ret = plugin_table_schema_set_ex_data_schema(table_schema, new_func, free_func, dup_func, argl, argp);
if (ret < 0) {
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return -1;
}
struct table_runtime *table_rt = NULL;
if (maat_instance->maat_rt != NULL) {
table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id);
plugin_table_runtime_commit_ex_data_schema(table_rt, table_schema);
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return 0;
}
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
const char *key, size_t key_len)
{
struct maat_runtime *maat_rt = maat_instance->maat_rt;
if (NULL == maat_rt) {
return NULL;
}
struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id);
return plugin_table_runtime_get_ex_data(table_rt, table_schema, key, key_len);
}
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, int results[], size_t n_result,
struct maat_state *state)
@@ -87,16 +288,10 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
const char *data, size_t data_len, int results[], size_t *n_results,
struct maat_state *state)
{
#if 0
if (NULL == maat_instance || NULL == data ) {
return -1;
}
struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr;
struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id);
struct maat_runtime *maat_rt = maat_instance->maat_rt;
struct maat_table_runtime *table_rt = maat_table_runtime_get(maat_rt->table_rt_mgr, table_id);
#endif
return 0;
return table_runtime_scan_string(table_rt, thread_id, data, data_len, results, n_results);
}
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id)

201
src/maat_config_monitor.cpp
View File

@@ -13,9 +13,11 @@
#include <string.h>
#include <limits.h>
#include <sys/stat.h>
#include <dirent.h>
#include "maat_config_monitor.h"
#include "maat_utils.h"
#include "utils.h"
#define CM_MAX_TABLE_NUM 256
#define MAX_CONFIG_LINE (1024 * 16)
@@ -28,17 +30,6 @@ struct cm_table_info_t
char encryp_algorithm[NAME_MAX];
};
/*
@brief check if rule file updated
@retval 0 -> CONFIG_UPDATE_TYPE_NONE
1 -> CONFIG_UPDATE_TYPE_FULL
2 -> CONFIG_UPDATE_TYPE_INC
*/
int validate_update_happened(uint64_t current_version, const char *idx_dir, char **idx_paths[], size_t *n_idx_paths)
{
}
int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int size)
{
int ret = 0;
@@ -55,7 +46,7 @@ int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int si
,idx[i].cfg_path
,idx[i].encryp_algorithm);
//jump over empty line
if (!(ret==3||ret==4)||idx[i].cfg_num==0){
if (!(ret == 3 || ret == 4) || idx[i].cfg_num == 0) {
continue;
}
@@ -77,17 +68,15 @@ int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int si
return i;
}
const char* path2filename(const char*path)
const char *path2filename(const char *path)
{
int i=0;
for(i=strlen(path);i>0;i--)
{
if(path[i]=='/')
{
int i = 0;
for (i = strlen(path);i > 0;i--) {
if (path[i]=='/') {
break;
}
}
return path+i+1;
return path + i + 1;
}
char *read_nxt_line_from_buff(const char *buff, size_t buff_size, size_t *offset, char *line, int line_size)
@@ -96,9 +85,9 @@ char *read_nxt_line_from_buff(const char *buff, size_t buff_size, size_t *offset
const char* p;
//search for CRLF, aka CR '\r'(old Mac), LF '\n'(UNIX) or CRLF"\r\n" (Windows)
p = (const char*)memchr(buff+*offset, '\n', buff_size-*offset);
p = (const char *)memchr(buff + *offset, '\n', buff_size - *offset);
if (p == NULL) { // NOT "\n" or "\r\n"
p = (const char*)memchr(buff+*offset, '\r', buff_size-*offset);
p = (const char *)memchr(buff + *offset, '\r', buff_size - *offset);
}
if (p != NULL) { //point to next character
@@ -108,7 +97,7 @@ char *read_nxt_line_from_buff(const char *buff, size_t buff_size, size_t *offset
}
this_offset = p - (buff + *offset);
memcpy(line, buff + *offset, MIN(this_offset, line_size-1));
memcpy(line, buff + *offset, MIN(this_offset, line_size - 1));
*offset += this_offset;
line[MIN(this_offset, line_size - 1)] = '\0';
@@ -122,7 +111,6 @@ int cm_read_table_file(struct cm_table_info_t* index,
{
int cfg_num = 0,i =0;
int ret = 0;
char error_string[NAME_MAX];
char line[MAX_CONFIG_LINE]={0};
char *ret_str=NULL;
char *table_file_buff=NULL;
@@ -144,16 +132,15 @@ int cm_read_table_file(struct cm_table_info_t* index,
}
for (i = 0; i < cfg_num; i++) {
line[sizeof(line)-1]='\0';
line[sizeof(line) - 1] = '\0';
ret_str=read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
if(ret_str==NULL) {
ret_str = read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
if (ret_str == NULL) {
//log_error
break;
}
if(line[sizeof(line)-1]!='\0') {
if(line[sizeof(line) - 1] != '\0') {
//log_error
continue;
}
@@ -169,32 +156,174 @@ int cm_read_table_file(struct cm_table_info_t* index,
return 0;
}
void config_monitor_traverse(uint64_t current_version, const char *idx_dir,
void (*start_fn)(uint64_t, int, void *),
//replacement of glibc scandir, to adapt dictator malloc wrap
#define ENLARGE_STEP 1024
int my_scandir(const char *dir, struct dirent ***namelist,
int(*filter)(const struct dirent *),
int(*compar)(const void *, const void *))
{
int n = 0;
int DIR_ENT_SIZE = ENLARGE_STEP;
struct dirent entry, *result;
if ((NULL == dir) || (NULL == namelist)) {
return -1;
}
DIR *od = opendir(dir);
if (NULL == od) {
return -1;
}
struct dirent **list = ALLOC(struct dirent *, DIR_ENT_SIZE);
while (0 == readdir_r(od, &entry, &result)) {
if (NULL == result) {
break;
}
if (filter && !filter(&entry)) {
continue;
}
struct dirent *p = ALLOC(struct dirent, 1);
memcpy((void *)p, (void *)(&entry), sizeof(struct dirent));
list[n] = p;
n++;
if (n >= DIR_ENT_SIZE) {
DIR_ENT_SIZE += ENLARGE_STEP;
list = (struct dirent **)realloc((void*)list, DIR_ENT_SIZE * sizeof(struct dirent *));
}
}
closedir(od);
*namelist = list;
if (compar) {
qsort((void *)*namelist, n, sizeof(struct dirent *), compar);
}
return n;
}
int filter_fn(const struct dirent *ent)
{
return (strncmp(ent->d_name,"full_config_index",strlen("full_config_index")) == 0 ||
strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0);
}
int get_new_idx_path(long long current_version, const char *file_dir, char ***idx_path, size_t *idx_num)
{
struct dirent **namelist = NULL;
int update_type = CM_UPDATE_TYPE_NONE;
int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort);
if (n < 0) {
//log_error("scan dir error");
return update_type;
}
long long latest_full_version = 0;
long long latest_inc_version = 0;
int full_file_idx = 0;
int *inc_file_idx = ALLOC(int, n);
int inc_idx_num = 0;
int i = 0;
for (i = 0; i < n; i++) {
long long config_seq = 0;
char update_str[32] = {0};
if ((strcmp(namelist[i]->d_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0)) {
continue;
}
if (strlen(namelist[i]->d_name) > 42) {
//log_error("config file %s filename too long,should like full_config_index.00000000000000000001",namelist[i]->d_name);
continue;
}
int ret = sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld", update_str, &config_seq);
if (ret != 2) {
//log_error("config file %s filename error,should like full_config_index.00000000000000000001",namelist[i]->d_name);
continue;
}
if (strncasecmp(update_str, "full", strlen(update_str)) == 0) {
if (config_seq > latest_full_version) {
latest_full_version = config_seq;
full_file_idx = i;
}
} else if(strncasecmp(update_str,"inc",strlen(update_str))==0) {
if (config_seq > current_version) {
inc_file_idx[inc_idx_num] = i;
inc_idx_num++;
if (config_seq > latest_inc_version) {
latest_inc_version = config_seq;
}
}
} else {
//log_error("config file %s,not full or inc config",namelist[i]->d_name);
}
}
size_t path_len = 0;
//full update
if (latest_full_version > current_version) {
*idx_path = (char **)malloc(sizeof(char **));
path_len = strlen(file_dir) + strlen(namelist[full_file_idx]->d_name) + 1 + 1;
(*idx_path)[0] = (char *)malloc(path_len);
snprintf((*idx_path)[0], path_len, "%s/%s", file_dir, namelist[full_file_idx]->d_name);
*idx_num = 1;
update_type = CM_UPDATE_TYPE_FULL;
} else if (latest_inc_version > current_version) {
//inc update,it's possible that do inc after full update in this function,but we'll process it at next loop.
*idx_path = (char **)malloc(sizeof(char **) * inc_idx_num);
for (i = 0; i < inc_idx_num; i++) {
path_len = strlen(file_dir) + strlen(namelist[inc_file_idx[i]]->d_name) + 1 + 1;
(*idx_path)[i] = (char *)malloc(path_len);
snprintf((*idx_path)[i], path_len, "%s/%s", file_dir, namelist[inc_file_idx[i]]->d_name);
}
*idx_num = inc_idx_num;
update_type = CM_UPDATE_TYPE_INC;
} else {
update_type = CM_UPDATE_TYPE_NONE;
}
free(inc_file_idx);
for (i = 0; i < n; i++) {
free(namelist[i]);
}
free(namelist);
return update_type;
}
void config_monitor_traverse(long long current_version, const char *idx_dir,
void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *),
void *u_param)
{
size_t i = 0;
uint64_t new_version = 0;
long long new_version = 0;
char **idx_path_array = NULL;
size_t idx_path_num = 0;
struct cm_table_info_t table_array[CM_MAX_TABLE_NUM];
memset(table_array, 0, sizeof(table_array));
int update_type = validate_update_happened(current_version, idx_dir, &idx_path_array, &idx_path_num);
if (update_type != CONFIG_UPDATE_TYPE_NONE) {
int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num);
if (update_type != CM_UPDATE_TYPE_NONE) {
for (i = 0; i < idx_path_num; i++) {
int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM);
if (table_num < 0) {
//log_error
//log_error luis
break;
}
char str_not_care[256] = {0};
const char* table_filename = path2filename(idx_path_array[i]);
sscanf(table_filename,"%[a-zA-Z]_config_index.%lld", str_not_care, &new_version);
const char *table_filename = path2filename(idx_path_array[i]);
sscanf(table_filename, "%[a-zA-Z]_config_index.%lld", str_not_care, &new_version);
if (start_fn != NULL) {
start_fn(new_version, update_type, u_param);
}

161
src/maat_ex_data.cpp
View File

@@ -9,53 +9,50 @@
*/
#include <stddef.h>
#include <stdio.h>
#include "uthash/uthash.h"
#include "uthash/utarray.h"
#include "maat_rhash.h"
#include "rcu_hash.h"
#include "utils.h"
#include "maat_utils.h"
typedef void ex_data_schema_new_func_t(int table_id, const char *key, const char *table_line, void* ad, long argl, void *argp);
typedef void ex_data_schema_free_func_t(int table_id, void* ad, long argl, void *argp);
typedef void ex_data_schema_dup_func_t(int table_id, void *to, void *from, long argl, void *argp);
/*
struct ex_data_schema
{
ex_data_schema_new_func_t* new_func;
ex_data_schema_free_func_t* free_func;
ex_data_schema_dup_func_t* dup_func;
//Maat_plugin_EX_key2index_func_t* key2index_func;
long argl;
void *argp;
}; */
#include "maat_table_schema.h"
#include "maat_ex_data.h"
struct ex_data_runtime {
UT_array *cache_rows;
size_t cache_row_num;
size_t cache_size;
struct rhash_table *htable;
//const struct ex_data_schema* ex_schema;
struct rcu_hash_table *htable;
struct ex_data_schema *ex_schema;
int table_id;
};
void cache_row_free(void*p)
void cache_row_free(void *p)
{
free(*(char**)p);
free(*(char **)p);
}
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
struct ex_data_runtime *ex_data_runtime_new(void (* data_free)(void *data))
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn)
{
struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1);
utarray_new(ex_data_rt->cache_rows, &ut_cache_row_icd);
ex_data_rt->htable = rhash_table_create(data_free);
ex_data_rt->htable = rcu_hash_new(data_free_fn);
ex_data_rt->table_id = table_id;
return ex_data_rt;
}
void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt)
{
if (NULL == ex_data_rt) {
return;
}
if (ex_data_rt->cache_rows != NULL) {
utarray_free(ex_data_rt->cache_rows);
ex_data_rt->cache_rows=NULL;
@@ -63,8 +60,128 @@ void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt)
}
if (ex_data_rt->htable != NULL) {
rhash_table_destroy(ex_data_rt->htable);
rcu_hash_free(ex_data_rt->htable);
}
free(ex_data_rt);
}
void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt)
{
rcu_hash_commit(ex_data_rt->htable);
}
void ex_data_runtime_cache_row_put(struct ex_data_runtime *ex_data_rt, const char *row)
{
size_t len = strlen(row) + 1;
char* row_copy = ALLOC(char, len);
memcpy(row_copy, row, len);
ex_data_rt->cache_size += len;
utarray_push_back(ex_data_rt->cache_rows, &row_copy);
ex_data_rt->cache_row_num++;
}
const char *ex_data_runtime_cached_row_get(struct ex_data_runtime *ex_data_rt, size_t index)
{
const char** row = NULL;
row = (const char **)utarray_eltptr(ex_data_rt->cache_rows, index);
return *row;
}
void ex_data_runtime_clear_cached_row(struct ex_data_runtime *ex_data_rt)
{
utarray_free(ex_data_rt->cache_rows);
ex_data_rt->cache_rows = NULL;
ex_data_rt->cache_row_num = 0;
ex_data_rt->cache_size = 0;
}
size_t ex_data_runtime_cached_row_count(struct ex_data_runtime *ex_data_rt)
{
if (NULL == ex_data_rt) {
return 0;
}
return ex_data_rt->cache_row_num;
}
void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt)
{
utarray_free(ex_data_rt->cache_rows);
ex_data_rt->cache_rows = NULL;
ex_data_rt->cache_row_num = 0;
ex_data_rt->cache_size = 0;
}
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema)
{
ex_data_rt->ex_schema = schema;
}
void ex_data_runtime_set_user_ctx(struct ex_data_runtime *ex_data_rt, void *user_ctx)
{
rcu_hash_set_user_ctx(ex_data_rt->htable, user_ctx);
}
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len)
{
void *ex_data = NULL;
struct ex_data_schema *ex_schema = ex_data_rt->ex_schema;
ex_schema->new_func(ex_data_rt->table_id, key, row, &ex_data, ex_schema->argl, ex_schema->argp);
return ex_data;
}
void ex_data_runtime_add_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, void *data)
{
void *tmp_data = rcu_hash_find(ex_data_rt->htable, key, key_len);
if (tmp_data != NULL) {
return;
}
rcu_hash_add(ex_data_rt->htable, key, key_len, data);
}
void ex_data_runtime_del_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len)
{
void *tmp_data = rcu_hash_find(ex_data_rt->htable, key, key_len);
if (NULL == tmp_data) {
fprintf(stderr, "ex data del error: no such key:%s", key);
return;
}
rcu_hash_del(ex_data_rt->htable, key, key_len);
}
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len)
{
void *ex_data = rcu_hash_find(ex_data_rt->htable, key, key_len);
if (NULL == ex_data) {
return NULL;
}
void *dup_ex_data = NULL;
ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, &ex_data,
ex_data_rt->ex_schema->argl, ex_data_rt->ex_schema->argp);
return dup_ex_data;
}
size_t ex_data_runtime_ex_data_count(struct ex_data_runtime *ex_data_rt)
{
return rcu_hash_count(ex_data_rt->htable);
}
size_t ex_data_runtime_list_updating_ex_data(struct ex_data_runtime *ex_data_rt, void ***ex_data_array)
{
if (NULL == ex_data_rt->ex_schema) {
return 0;
}
return rcu_hash_list_updating_data(ex_data_rt->htable, ex_data_array);
}
int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt)
{
return rcu_hash_updating_flag(ex_data_rt->htable);
}

2
src/maat_garbage_collection.cpp
View File

@@ -8,7 +8,7 @@
***********************************************************************************************
*/
#include "maat_utils.h"
#include "utils.h"
#include <time.h>
#include <stdlib.h>

2
src/maat_kv.cpp
View File

@@ -12,7 +12,7 @@
#include <ctype.h>
#include "uthash/uthash.h"
#include "maat_utils.h"
#include "utils.h"
#define MAAT_KV_MAX_KEY_LEN 512

164
src/maat_rule.cpp
View File

@@ -9,92 +9,179 @@
*/
#include <unistd.h>
#include <stdio.h>
#include <pthread.h>
#include <limits.h>
#include <string.h>
#include <sys/prctl.h>
#include <assert.h>
#include "maat_rule.h"
#include "maat_config_monitor.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_table_runtime.h"
#include "maat_table_schema.h"
struct maat_runtime* create_maat_runtime(uint64_t version, struct maat *maat_instance)
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance)
{
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
maat_rt->version = version;
maat_rt->table_rt_mgr = maat_table_runtime_manager_create(maat_instance->table_mgr);
maat_rt->max_table_num = maat_table_manager_get_size(maat_instance->table_mgr);
maat_rt->table_rt_mgr = table_runtime_manager_create(maat_instance->table_schema_mgr,
maat_instance->nr_worker_thread,
maat_instance->garbage_bin);
maat_rt->max_table_num = table_schema_manager_get_size(maat_instance->table_schema_mgr);
maat_rt->max_thread_num = maat_instance->nr_worker_thread;
return maat_rt;
}
void update_maat_runtime(struct maat_runtime *maat_rt)
void maat_runtime_commit(struct maat_runtime *maat_rt)
{
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
struct maat_table_runtime *table_rt = maat_table_runtime_get(maat_rt->table_rt_mgr, i);
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i);
if (NULL == table_rt) {
continue;
}
enum maat_table_type table_type = maat_table_runtime_get_type(table_rt);
table_runtime_commit(table_rt, maat_rt->max_thread_num);
}
maat_rt->last_update_time = time(NULL);
}
void destroy_maat_runtime(struct maat_runtime *maat_rt)
void maat_runtime_destroy(struct maat_runtime *maat_rt)
{
if (NULL == maat_rt) {
return;
}
if (maat_rt->table_rt_mgr != NULL) {
table_runtime_manager_destroy(maat_rt->table_rt_mgr);
maat_rt->table_rt_mgr = NULL;
}
free(maat_rt);
}
void maat_start_cb(uint64_t new_version, int update_type, void *u_param)
int maat_runtime_updating_flag(struct maat_runtime *maat_rt)
{
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i);
if (NULL == table_rt) {
continue;
}
int flag = table_runtime_updating_flag(table_rt);
if (1 == flag) {
return 1;
}
}
return 0;
}
void maat_start_cb(long long new_version, int update_type, void *u_param)
{
struct maat *maat_instance = (struct maat *)u_param;
if (update_type == CONFIG_UPDATE_TYPE_FULL) {
maat_instance->rebuilding_maat_rt = create_maat_runtime(new_version, maat_instance);
if (update_type == CM_UPDATE_TYPE_FULL) {
maat_instance->creating_maat_rt = maat_runtime_create(new_version, maat_instance);
} else {
maat_instance->maat_version = new_version;
}
table_schema_manager_all_plugin_cb_start(maat_instance->table_schema_mgr, update_type);
}
int maat_update_cb(const char* table_name, const char* line, void *u_param)
int maat_update_cb(const char *table_name, const char *line, void *u_param)
{
struct maat *maat_instance =(struct maat *)u_param;
struct maat_runtime* maat_rt = NULL;
int table_id = maat_table_manager_get_table_id(maat_instance->table_mgr, table_name);
struct maat_table_schema* ptable = maat_table_manager_get_table_desc(maat_instance->table_mgr, table_id);
int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, table_name);
struct table_schema* table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
if (maat_instance->rebuilding_maat_rt != NULL) {
maat_rt = maat_instance->rebuilding_maat_rt;
if (maat_instance->creating_maat_rt != NULL) {
maat_rt = maat_instance->creating_maat_rt;
} else {
maat_rt = maat_instance->maat_rt;
}
//TODO: update rule for table_schema
update_maat_runtime();
struct table_item *table_item = table_schema_line_to_item(line, table_schema);
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id);
table_runtime_update(table_rt, table_schema, line, table_item);
free(table_item);
return 0;
}
uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt)
{
uint32_t total = 0;
struct table_runtime *table_rt = NULL;
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
table_rt = table_runtime_get(maat_rt->table_rt_mgr, i);
if (table_rt != NULL) {
total += table_runtime_rule_count(table_rt);
}
}
return total;
}
void maat_finish_cb(void *u_param)
{
struct maat *maat_instance = (struct maat *)u_param;
if (maat_instance->latest_maat_rt != NULL) {
update_maat_runtime(maat_instance->latest_maat_rt);
table_schema_manager_all_plugin_cb_finish(maat_instance->table_schema_mgr);
if (maat_instance->creating_maat_rt != NULL) {
maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt);
maat_runtime_commit(maat_instance->creating_maat_rt);
fprintf(stdout, "Full config version %llu load %d entries complete",
maat_instance->creating_maat_rt->version,
maat_instance->creating_maat_rt->rule_num);
} else if (maat_instance->maat_rt != NULL) {
maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt);
maat_instance->maat_rt->version = maat_instance->maat_version;
maat_runtime_commit(maat_instance->maat_rt);
fprintf(stdout, "Inc config version %llu load %d entries complete",
maat_instance->maat_rt->version,
maat_instance->maat_rt->rule_num);
}
//
}
void *rule_monitor_loop(void *arg)
{
char err_str[NAME_MAX] = {0};
/* Defined by prctl: The name can be up to 16 bytes long, and should
be null terminated if it contains fewer bytes. */
char maat_name[16] = {0};
struct maat *maat_instance = (struct maat *)arg;
if (strlen(maat_instance->instance_name) > 0) {
snprintf(maat_name, sizeof(maat_name), "MAAT_%s", maat_instance->instance_name);
} else {
snprintf(maat_name, sizeof(maat_name), "MAAT");
}
int ret = prctl(PR_SET_NAME, (unsigned long long)maat_name, NULL, NULL, NULL);
assert(ret >= 0);
pthread_mutex_lock(&(maat_instance->background_update_mutex));
/* if deferred load on */
if (maat_instance->deferred_load != 0) {
fprintf(stdout, "Deferred Loading ON, updating in %s.", __func__);
maat_read_full_config(maat_instance);
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
while (maat_instance->is_running) {
usleep(1000 * 1000);
usleep(maat_instance->rule_update_checking_interval_ms * 1000);
if( 0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
switch (maat_instance->rule_import_type) {
case RULE_IMPORT_TYPE_IRIS:
switch (maat_instance->input_mode) {
case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version,
maat_instance->iris_ctx.inc_dir,
maat_start_cb,
@@ -106,25 +193,38 @@ void *rule_monitor_loop(void *arg)
break;
}
if (maat_instance->latest_maat_rt != NULL) {
if (maat_instance->creating_maat_rt != NULL) {
struct maat_runtime *old_maat_rt = maat_instance->maat_rt;
maat_instance->maat_rt = maat_instance->latest_maat_rt;
maat_instance->maat_rt = maat_instance->creating_maat_rt;
if (old_maat_rt != NULL) {
maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy);
}
maat_instance->latest_maat_rt = NULL;
maat_instance->creating_maat_rt = NULL;
maat_instance->maat_version = maat_instance->maat_rt->version;
maat_instance->last_full_version = maat_instance->maat_rt->version;
}
if (maat_instance->maat_rt != NULL) {
update_maat_runtime(maat_instance->maat_rt);
int updating_flag = maat_runtime_updating_flag(maat_instance->maat_rt);
time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time;
if ((updating_flag > 0) && (time_window >= maat_instance->rule_effect_interval_ms / 1000)) {
maat_runtime_commit(maat_instance->maat_rt);
}
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
}
maat_garbage_collect_routine(maat_instance->garbage_bin);
}
maat_runtime_destroy(maat_instance->maat_rt);
maat_garbage_bin_free(maat_instance->garbage_bin);
table_schema_manager_destroy(maat_instance->table_schema_mgr);
free(maat_instance);
return NULL;
}

587
src/maat_table_runtime.cpp
View File

@@ -8,55 +8,108 @@
***********************************************************************************************
*/
#include <stdio.h>
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include <hs/hs.h>
#include "utils.h"
#include "maat_utils.h"
#include "maat_table_runtime.h"
#include "uthash/uthash.h"
#include "maat_ex_data.h"
#include "adapter_hs.h"
#include "rcu_hash.h"
#include "IPMatcher.h"
struct maat_expr_runtime {
struct adapter_hs *hs;
//rcu_hash *htable;
#define MAAT_MAX_EXPR_ITEM_NUM 8
struct plugin_ex_data {
void *data;
void *user_data;
};
struct maat_ip_plugin_runtime {
//struct ip_matcher* ip_matcher;
struct plugin_user_ctx {
int table_id;
struct ex_data_schema *ex_schema;
};
struct expr_runtime {
enum scan_mode scan_mode;
struct adapter_hs *hs;
struct adapter_hs_stream *hs_stream;
struct rcu_hash_table *htable;
};
struct ip_runtime {
struct ip_matcher *ip_matcher;
struct rcu_hash_table *htable;
};
struct plugin_runtime {
uint64_t acc_line_num;
struct ex_data_runtime *ex_data_rt;
};
struct ip_plugin_runtime {
struct ip_matcher* ip_matcher;
struct ex_data_runtime* ex_data_rt;
};
struct maat_table_runtime {
enum maat_table_type table_type; // table schema已有type??
struct table_runtime {
uint32_t rule_num;
uint32_t updating_rule_num;
enum table_type table_type;
union {
struct maat_expr_runtime *expr_rt;
struct maat_ip_plugin_runtime *ip_plugin_rt;
struct expr_runtime expr_rt;
struct ip_runtime ip_rt;
struct plugin_runtime plugin_rt;
struct ip_plugin_runtime ip_plugin_rt;
};
struct maat_garbage_bin *ref_garbage_bin;
//ex_data_rt
//table相关指针
};
struct maat_table_runtime_manager {
struct maat_table_runtime **table_rt;
struct table_runtime_manager {
struct table_runtime **table_rt;
size_t n_table_rt;
struct maat_garbage_bin *garbage_bin;
};
struct maat_table_runtime *table_runtime_new(enum maat_table_type table_type, int max_thread_num, struct maat_garbage_bin* bin)
void plugin_ex_data_free(void *user_ctx, void *data)
{
struct maat_table_runtime *table_rt = ALLOC(struct maat_table_runtime, 1);
table_rt->table_type = table_type;
switch (table_type) {
struct plugin_user_ctx *ctx = (struct plugin_user_ctx *)user_ctx;
long argl = ctx->ex_schema->argl;
void *argp = ctx->ex_schema->argp;
ctx->ex_schema->free_func(ctx->table_id, &data, argl, argp);
}
void expr_ex_data_free(void *user_ctx, void *data)
{
free(data);
}
struct table_runtime *table_runtime_new(struct table_schema *table_schema, int max_thread_num, struct maat_garbage_bin* bin)
{
int table_id = table_schema_get_table_id(table_schema);
struct table_runtime *table_rt = ALLOC(struct table_runtime, 1);
table_rt->ref_garbage_bin = bin;
table_rt->table_type = table_schema_get_table_type(table_schema);
switch (table_rt->table_type) {
case TABLE_TYPE_EXPR:
table_rt->expr_rt->ex_data_rt = ex_data_runtime_new(NULL);
table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free);
table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema);
break;
case TABLE_TYPE_PLUGIN:
table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, plugin_ex_data_free);
break;
case TABLE_TYPE_IP_PLUGIN:
table_rt->ip_plugin_rt->ex_data_rt = ex_data_runtime_new(NULL);
table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, plugin_ex_data_free);
break;
default:
break;
@@ -65,44 +118,51 @@ struct maat_table_runtime *table_runtime_new(enum maat_table_type table_type, in
return table_rt;
}
void table_runtime_free(struct maat_table_runtime * table_rt)
void table_runtime_free(struct table_runtime * table_rt)
{
switch (table_rt->table_type)
{
case TABLE_TYPE_EXPR:
ex_data_runtime_free(table_rt->expr_rt->ex_data_rt);
break;
case TABLE_TYPE_IP_PLUGIN:
ex_data_runtime_free(table_rt->ip_plugin_rt->ex_data_rt);
default:
break;
switch (table_rt->table_type) {
case TABLE_TYPE_EXPR:
adapter_hs_destroy(table_rt->expr_rt.hs);
rcu_hash_free(table_rt->expr_rt.htable);
break;
case TABLE_TYPE_PLUGIN:
ex_data_runtime_free(table_rt->plugin_rt.ex_data_rt);
break;
case TABLE_TYPE_IP_PLUGIN:
ip_matcher_free(table_rt->ip_plugin_rt.ip_matcher);
ex_data_runtime_free(table_rt->ip_plugin_rt.ex_data_rt);
break;
default:
break;
}
free(table_rt);
}
struct maat_table_runtime_manager *maat_table_runtime_manager_create(struct maat_table_manager *table_mgr, int max_thread_num, struct maat_garbage_bin* garbage_bin)
struct table_runtime_manager *
table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num,
struct maat_garbage_bin* garbage_bin)
{
if (NULL == table_mgr) {
if (NULL == table_schema_mgr) {
return NULL;
}
struct maat_table_runtime_manager *table_rt_mgr = ALLOC(struct maat_table_runtime_manager, 1);
table_rt_mgr->n_table_rt = maat_table_manager_get_size(table_mgr);
table_rt_mgr->table_rt = ALLOC(struct maat_table_runtime *, table_rt_mgr->n_table_rt);
struct table_runtime_manager *table_rt_mgr = ALLOC(struct table_runtime_manager, 1);
table_rt_mgr->n_table_rt = table_schema_manager_get_size(table_schema_mgr);
table_rt_mgr->table_rt = ALLOC(struct table_runtime *, table_rt_mgr->n_table_rt);
for (size_t i = 0; i < table_rt_mgr->n_table_rt; i++) {
enum maat_table_type table_type = maat_table_manager_get_table_type(table_mgr, i);
if (table_type == TABLE_TYPE_MAX) {
struct table_schema *table_schema = table_schema_get(table_schema_mgr, i);
if (NULL == table_schema) {
continue;
}
table_rt_mgr->table_rt[i] = table_runtime_new(table_type, max_thread_num, garbage_bin);
table_rt_mgr->table_rt[i] = table_runtime_new(table_schema, max_thread_num, garbage_bin);
}
return table_rt_mgr;
}
void maat_table_runtime_manager_destroy(struct maat_table_runtime_manager *table_rt_mgr)
void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr)
{
for(size_t i = 0; i < table_rt_mgr->n_table_rt; i++) {
table_runtime_free(table_rt_mgr->table_rt[i]);
@@ -114,13 +174,462 @@ void maat_table_runtime_manager_destroy(struct maat_table_runtime_manager *table
free(table_rt_mgr);
}
struct maat_table_runtime *maat_table_runtime_get(struct maat_table_runtime_manager *table_rt_mgr, int table_id)
struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id)
{
assert(table_id < (int)table_rt_mgr->n_table_rt);
return table_rt_mgr->table_rt[table_id];
}
enum maat_table_type maat_table_runtime_get_type(struct maat_table_runtime* table_rt)
size_t table_runtime_rule_count(struct table_runtime *table_rt)
{
return table_rt->rule_num;
}
enum table_type table_runtime_get_type(struct table_runtime* table_rt)
{
return table_rt->table_type;
}
int table_runtime_scan_string(struct table_runtime* table_rt, int thread_id, const char *data, size_t data_len,
int result[], size_t *n_result)
{
return adapter_hs_scan(table_rt->expr_rt.hs, thread_id, data, data_len, result, n_result);
}
void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id)
{
struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(table_rt->expr_rt.hs, thread_id);
table_rt->expr_rt.hs_stream = hs_stream;
}
int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len,
int result[], size_t *n_result)
{
return adapter_hs_scan_stream(table_rt->expr_rt.hs_stream, data, data_len, result, n_result);
}
void table_runtime_stream_close(struct table_runtime *table_rt)
{
adapter_hs_stream_close(table_rt->expr_rt.hs_stream);
table_rt->expr_rt.hs_stream = NULL;
}
struct ip_rule *ip_plugin_item_to_ip_rule(struct ip_plugin_item *ip_plugin_item)
{
return NULL;
}
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item)
{
size_t i = 0;
size_t sub_expr_cnt = 0;
char *pos = NULL;
char *saveptr = NULL;
char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM];
and_expr_t *expr_rule = ALLOC(and_expr_t, 1);
switch (expr_item->expr_type) {
case EXPR_TYPE_AND:
case EXPR_TYPE_REGEX:
for (i = 0, pos = expr_item->keywords; ; i++, pos = NULL) {
char *tmp = strtok_r_esc(pos, '&', &saveptr);
if (NULL == tmp) {
break;
}
if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
fprintf(stderr, "item_id:%d too many patterns", expr_item->item_id);
return NULL;
}
sub_key_array[i] = tmp;
if (expr_item->expr_type == EXPR_TYPE_REGEX) {
sub_key_array[i] = str_unescape_and(sub_key_array[i]);
} else {
sub_key_array[i] = str_unescape(sub_key_array[i]);
}
}
sub_expr_cnt = i;
break;
case EXPR_TYPE_STRING:
sub_expr_cnt = 1;
sub_key_array[0] = expr_item->keywords;
sub_key_array[0] = str_unescape(sub_key_array[0]);
break;
default:
break;
}
for (i = 0; i < sub_expr_cnt; i++) {
expr_rule->expr_id = expr_item->item_id;
expr_rule->patterns[i].pat = ALLOC(char, strlen(sub_key_array[i]));
memcpy(expr_rule->patterns[i].pat, sub_key_array[i], strlen(sub_key_array[i]));
expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]);
expr_rule->patterns[i].type = expr_item->expr_type;
}
expr_rule->n_patterns = sub_expr_cnt;
return expr_rule;
}
void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len,
and_expr_t *expr_rule, int is_valid)
{
void *data = NULL;
if (is_valid == 0) {
//delete
data = rcu_hash_find(expr_rt->htable, key, key_len);
if (NULL == data) {
//log_error("the key:%s not exist, so can't be deleted.");
return;
}
rcu_hash_del(expr_rt->htable, key, key_len);
} else {
//add
data = rcu_hash_find(expr_rt->htable, key, key_len);
if (data != NULL) {
//log_error("the key:%s already exist, so can't be added.");
return;
}
and_expr_t *data = ALLOC(and_expr_t, 1);
memcpy(data, expr_rule, sizeof(and_expr_t));
rcu_hash_add(expr_rt->htable, key, key_len, (void *)data);
}
}
void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_schema *table_schema,
const char *row, char *key, size_t key_len, int is_valid)
{
struct plugin_ex_data *ex_data = ALLOC(struct plugin_ex_data, 1);
ex_data->data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema);
size_t cb_count = plugin_table_schema_callback_count(table_schema);
/* already set plugin_table_schema's ex_data_schema */
if (1 == set_flag) {
if (is_valid == 0) {
// delete
ex_data_runtime_del_ex_data(plugin_rt->ex_data_rt, key, key_len);
} else {
// add
ex_data_runtime_add_ex_data(plugin_rt->ex_data_rt, key, key_len, (void *)ex_data);
}
}
/* plugin table schema has callback */
if (cb_count > 0) {
plugin_table_schema_all_cb_update(table_schema, row);
}
if ((0 == set_flag) && (0 == cb_count)) {
ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row);
}
plugin_rt->acc_line_num++;
}
void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct table_schema *table_schema,
const char *row, char *key, size_t key_len,
struct ip_rule *ip_rule, int is_valid)
{
struct plugin_ex_data *ex_data = ALLOC(struct plugin_ex_data, 1);
struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt;
ex_data->data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len);
ex_data->user_data = ip_rule;
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema);
if (1 == set_flag) {
if (0 == is_valid) {
//delete
ex_data_runtime_del_ex_data(ex_data_rt, key, key_len);
} else {
//add
ex_data_runtime_add_ex_data(ip_plugin_rt->ex_data_rt, key, key_len, ex_data);
}
} else {
ex_data_runtime_cache_row_put(ip_plugin_rt->ex_data_rt, row);
}
}
void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, const char *row, struct table_item *table_item)
{
int is_valid = -1;
char *key = NULL;
size_t key_len = 0;
and_expr_t *expr_rule = NULL;
struct ip_rule *ip_rule = NULL;
switch (table_rt->table_type) {
case TABLE_TYPE_EXPR:
is_valid = table_item->expr_item.is_valid;
expr_rule = expr_item_to_expr_rule(&table_item->expr_item);
key = (char *)&(table_item->expr_item.item_id);
expr_runtime_update_row(&(table_rt->expr_rt), key, sizeof(int), expr_rule, is_valid);
free(expr_rule);
break;
case TABLE_TYPE_PLUGIN:
is_valid = table_item->plugin_item.is_valid;
key = table_item->plugin_item.key;
key_len = table_item->plugin_item.key_len;
plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, key, key_len, is_valid);
break;
case TABLE_TYPE_IP_PLUGIN:
is_valid = table_item->ip_plugin_item.is_valid;
ip_rule = ip_plugin_item_to_ip_rule(&table_item->ip_plugin_item);
key = (char *)&(table_item->ip_plugin_item.item_id);
ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, row, key, sizeof(int), ip_rule, is_valid);
free(ip_rule);
break;
default:
break;
}
if (is_valid == 0) {
table_rt->rule_num--;
} else {
table_rt->rule_num++;
}
}
int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread)
{
struct expr_runtime *expr_rt = &(table_rt->expr_rt);
void **ex_data_array = NULL;
and_expr_t *rules = NULL;
size_t rule_cnt = 0;
int ret = 0;
rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array);
rules = ALLOC(and_expr_t, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(and_expr_t *)ex_data_array[i];
}
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
if (rule_cnt > 0) {
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, nr_worker_thread, rules, rule_cnt);
if (NULL == new_adapter_hs) {
ret = -1;
}
}
old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs;
maat_garbage_bagging(table_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy);
rcu_hash_commit(expr_rt->htable);
table_rt->rule_num = rcu_hash_count(expr_rt->htable);
free(rules);
free(ex_data_array);
return ret;
}
int plugin_runtime_commit(struct table_runtime *table_rt)
{
ex_data_runtime_commit(table_rt->plugin_rt.ex_data_rt);
table_rt->rule_num = ex_data_runtime_ex_data_count(table_rt->plugin_rt.ex_data_rt);
return 0;
}
int ip_plugin_runtime_commit(struct table_runtime *table_rt)
{
struct plugin_ex_data **ex_data_array = NULL;
struct ip_rule *rules = NULL;
size_t rule_cnt = 0;
int ret = 0;
struct ip_plugin_runtime *ip_plugin_rt = &(table_rt->ip_plugin_rt);
rule_cnt = ex_data_runtime_list_updating_ex_data(ip_plugin_rt->ex_data_rt, (void ***)&ex_data_array);
rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct ip_rule *)ex_data_array[i]->user_data;
}
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
size_t mem_used = 0;
if (rule_cnt > 0) {
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
ret = -1;
}
}
old_ip_matcher = ip_plugin_rt->ip_matcher;
ip_plugin_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ip_plugin_rt->ex_data_rt);
table_rt->rule_num = ex_data_runtime_ex_data_count(ip_plugin_rt->ex_data_rt);
free(rules);
free(ex_data_array);
return ret;
}
int expr_runtime_updating_flag(struct expr_runtime *expr_rt)
{
return rcu_hash_updating_flag(expr_rt->htable);
}
int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt)
{
return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt);
}
int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt)
{
return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt);
}
int table_runtime_updating_flag(struct table_runtime *table_rt)
{
int updating_flag = 0;
switch (table_rt->table_type) {
case TABLE_TYPE_EXPR:
updating_flag = expr_runtime_updating_flag(&(table_rt->expr_rt));
break;
case TABLE_TYPE_PLUGIN:
updating_flag = plugin_runtime_updating_flag(&(table_rt->plugin_rt));
break;
case TABLE_TYPE_IP_PLUGIN:
updating_flag = ip_plugin_runtime_updating_flag(&(table_rt->ip_plugin_rt));
break;
default:
break;
}
return updating_flag;
}
void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread)
{
switch (table_rt->table_type) {
case TABLE_TYPE_EXPR:
expr_runtime_commit(table_rt, nr_worker_thread);
break;
case TABLE_TYPE_PLUGIN:
plugin_runtime_commit(table_rt);
case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_commit(table_rt);
break;
default:
break;
}
}
size_t plugin_table_runtime_cached_row_count(struct table_runtime* table_rt)
{
size_t row_count = 0;
struct ex_data_runtime *ex_data_rt = NULL;
switch (table_rt->table_type) {
case TABLE_TYPE_IP_PLUGIN:
ex_data_rt = table_rt->ip_plugin_rt.ex_data_rt;
break;
default:
break;
}
row_count = ex_data_runtime_cached_row_count(ex_data_rt);
return row_count;
}
const char* plugin_table_runtime_get_cached_row(struct table_runtime* table_rt, size_t row_seq)
{
const char *line = NULL;
struct ex_data_runtime *ex_data_rt = NULL;
switch (table_rt->table_type) {
case TABLE_TYPE_IP_PLUGIN:
ex_data_rt = table_rt->ip_plugin_rt.ex_data_rt;
break;
default:
break;
}
line = ex_data_runtime_cached_row_get(ex_data_rt, row_seq);
return line;
}
void *plugin_table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema,
const char *key, size_t key_len)
{
void *ex_data = NULL;
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema);
if (0 == set_flag) {
assert(0);
return NULL;
}
enum table_type table_type = table_schema_get_table_type(table_schema);
switch (table_type) {
case TABLE_TYPE_PLUGIN:
ex_data = ex_data_runtime_get_ex_data(table_rt->plugin_rt.ex_data_rt, key, key_len);
break;
case TABLE_TYPE_IP_PLUGIN:
break;
default:
break;
}
return ex_data;
}
struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt)
{
struct ex_data_runtime *ex_data_rt = NULL;
switch (table_rt->table_type) {
case TABLE_TYPE_PLUGIN:
ex_data_rt = table_rt->plugin_rt.ex_data_rt;
break;
case TABLE_TYPE_IP_PLUGIN:
ex_data_rt = table_rt->ip_plugin_rt.ex_data_rt;
break;
default:
break;
}
return ex_data_rt;
}
void plugin_table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema)
{
struct ex_data_schema *ex_data_schema = plugin_table_schema_get_ex_data_schema(table_schema);
struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt);
ex_data_runtime_set_schema(ex_data_rt, ex_data_schema);
struct plugin_user_ctx *user_ctx = ALLOC(struct plugin_user_ctx, 1);
user_ctx->table_id = table_schema_get_table_id(table_schema);
user_ctx->ex_schema = ex_data_schema;
ex_data_runtime_set_user_ctx(ex_data_rt, user_ctx);
size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt);
for (size_t i = 0; i < n_cached_row; i++) {
const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i);
switch (table_rt->table_type) {
case TABLE_TYPE_PLUGIN:
plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, NULL, 0, 1);
break;
case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, NULL, NULL, 0, NULL, 1);
break;
default:
break;
}
}
ex_data_runtime_clear_row_cache(ex_data_rt);
table_runtime_commit(table_rt, 0);
}

912
src/maat_table_schema.cpp
View File

File diff suppressed because it is too large Load Diff

170
src/maat_utils.cpp
View File

@@ -8,28 +8,174 @@
***********************************************************************************************
*/
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include "maat_utils.h"
int get_column_pos(sds line, int column_seq, size_t *offset, size_t *len)
char *maat_strdup(const char *s)
{
const char* seps=" \t";
char* saveptr=NULL, *subtoken=NULL, *str=NULL;
sds dup_line = sdsdup(line);
int i=0, ret=-1;
for (str = dup_line; ; str = NULL)
{
if (NULL == s) {
return NULL;
}
char *d = (char *)malloc(strlen(s) + 1);
memcpy(d, s, strlen(s) + 1);
return d;
}
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len)
{
const char *seps=" \t";
char *saveptr=NULL, *subtoken=NULL, *str=NULL;
char *dup_line = maat_strdup(line);
int i = 0, ret = -1;
for (str = dup_line; ; str = NULL) {
subtoken = strtok_r(str, seps, &saveptr);
if (subtoken == NULL)
break;
if(i==column_seq-1)
{
*offset=subtoken-dup_line;
*len=strlen(subtoken);
ret=0;
if (i == column_seq - 1) {
*offset = subtoken - dup_line;
*len = strlen(subtoken);
ret = 0;
break;
}
i++;
}
free(dup_line);
return ret;
}
int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *out_sz)
{
int ret = 0;
FILE *fp = NULL;
struct stat fstat_buf;
size_t read_size=0;
ret = stat(file_name, &fstat_buf);
if (ret != 0) {
return -1;
}
fp = fopen(file_name, "r");
if (fp == NULL) {
return -1;
}
*out_sz = fstat_buf.st_size;
*pp_out = (unsigned char *)calloc(1, *out_sz+1);
read_size = fread(*pp_out, 1, *out_sz, fp);
if (read_size != *out_sz) {
free(*pp_out);
pp_out = NULL;
return -1;
}
fclose(fp);
fp = NULL;
return 0;
}
static char *strchr_esc(char *s, const char delim)
{
char *token = NULL;
if (NULL == s) {
return NULL;
}
for (token = s; *token != '\0'; token++) {
if (*token == '\\') {
token++;
continue;
}
if(*token == delim) {
break;
}
}
if (*token == '\0') {
return NULL;
} else {
return token;
}
}
char *strtok_r_esc(char *s, const char delim, char **save_ptr)
{
char *token = NULL;
if (NULL == s) {
s = *save_ptr;
}
/* Scan leading delimiters. */
token = strchr_esc(s,delim);
if (NULL == token) {
*save_ptr = token;
return s;
}
/* Find the end of the token. */
*token = '\0';
token++;
*save_ptr = token;
return s;
}
char *str_unescape_and(char *s)
{
size_t i = 0;
size_t j = 0;
for (i = 0,j = 0; i < strlen(s); i++) {
if (s[i] == '\\' && s[i+1] == '&') {
s[j] = '&';
i++;
j++;
} else {
s[j] = s[i];
j++;
}
}
s[j] = '\0';
return s;
}
char *str_unescape(char *s)
{
size_t i=0;
size_t j=0;
for (i = 0, j = 0; i < strlen(s); i++) {
if (s[i] == '\\') {
switch (s[i+1]) {
case '&':
s[j] = '&';
break;
case 'b':
s[j] = ' ';//space,0x20;
break;
case '\\':
s[j] = '\\';
break;
default:
s[j] = s[i];
i--; //undo the followed i++
break;
}
i++;
j++;
} else {
s[j] = s[i];
j++;
}
}
s[j] = '\0';
return s;
}

168
src/rcu_hash.cpp
View File

@@ -11,12 +11,17 @@
#include <assert.h>
#include <pthread.h>
#include <unistd.h>
#include <sys/queue.h>
#include "rcu_hash.h"
#include "maat_utils.h"
#include "maat_garbage_collection.h"
#include "utils.h"
#define GARBAGE_DEFAULT_TIMEOUT 60
struct rcu_hash_garbage_bag {
void *garbage;
void (* garbage_free)(void *garbage);
TAILQ_ENTRY(rcu_hash_garbage_bag) entries;
};
TAILQ_HEAD(rcu_hash_garbage_q, rcu_hash_garbage_bag);
struct rcu_hash_table {
int is_updating;
@@ -26,8 +31,11 @@ struct rcu_hash_table {
struct rcu_hash_node *hashmap_a;
struct rcu_hash_node *hashmap_b;
void (* data_free)(void *data);
struct maat_garbage_bin *garbage_bin;
void (*data_free_fn)(void *user_ctx, void *data);
void *user_ctx;
struct rcu_hash_garbage_q garbage_q;
size_t garbage_q_len;
pthread_mutex_t update_mutex;
};
@@ -37,35 +45,67 @@ struct rcu_hash_node {
size_t key_len;
void *data; //table_runtime解析成两个成员
/* htable the node belongs to */
struct rcu_hash_table *htable;
UT_hash_handle hh_a;
UT_hash_handle hh_b;
};
void rcu_hash_node_free(struct rcu_hash_node *node, void (* data_free)(void *data))
void rcu_hash_garbage_queue_free(struct rcu_hash_garbage_q* garbage_q)
{
struct rcu_hash_garbage_bag *p = NULL;
while ((p = TAILQ_FIRST(garbage_q)) != NULL) {
p->garbage_free(p->garbage);
TAILQ_REMOVE(garbage_q, p, entries);
free(p);
}
}
size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable)
{
return htable->garbage_q_len;
}
void rcu_hash_garbage_bagging(struct rcu_hash_garbage_q* garbage_q, void* garbage, void (* func)(void *))
{
struct rcu_hash_garbage_bag *bag = ALLOC(struct rcu_hash_garbage_bag, 1);
bag->garbage = garbage;
bag->garbage_free = func;
TAILQ_INSERT_TAIL(garbage_q, bag, entries);
}
void rcu_hash_node_free(struct rcu_hash_node *node)
{
if (NULL == node) {
return;
}
if (node->key != NULL) {
free(node->key);
}
if (node->data != NULL) {
data_free(node->data);
node->htable->data_free_fn(node->htable->user_ctx, node->data);
}
free(node);
}
struct rcu_hash_table *rcu_hash_new(void (* data_free)(void *data))
struct rcu_hash_table *rcu_hash_new(rcu_hash_data_free_fn *free_fn)
{
if (NULL == data_free) {
if (NULL == free_fn) {
return NULL;
}
struct rcu_hash_table *htable = ALLOC(struct rcu_hash_table, 1);
htable->is_updating = 0;
htable->effective_hash = 'a';
htable->garbage_bin = maat_garbage_bin_new(GARBAGE_DEFAULT_TIMEOUT);
htable->data_free = data_free;
TAILQ_INIT(&htable->garbage_q);
htable->garbage_q_len = 0;
htable->data_free_fn = free_fn;
pthread_mutex_init(&htable->update_mutex, NULL);
return htable;
@@ -73,28 +113,36 @@ struct rcu_hash_table *rcu_hash_new(void (* data_free)(void *data))
void rcu_hash_free(struct rcu_hash_table *htable)
{
if (NULL == htable) {
return;
}
struct rcu_hash_node *tmp = NULL;
struct rcu_hash_node *item = NULL;
if (htable != NULL) {
if (htable->effective_hash == 'a') {
HASH_ITER(hh_a, htable->hashmap_a, item, tmp) {
HASH_DELETE(hh_a, htable->hashmap_a, item);
rcu_hash_node_free(item, htable->data_free);
rcu_hash_node_free(item);
}
} else {
HASH_ITER(hh_b, htable->hashmap_b, item, tmp) {
HASH_DELETE(hh_b, htable->hashmap_b, item);
rcu_hash_node_free(item, htable->data_free);
rcu_hash_node_free(item);
}
}
maat_garbage_bin_free(htable->garbage_bin);
rcu_hash_garbage_queue_free(&(htable->garbage_q));
pthread_mutex_destroy(&htable->update_mutex);
free(htable);
}
void rcu_hash_update_prepare(struct rcu_hash_table *htable)
void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx)
{
htable->user_ctx = user_ctx;
}
void rcu_hash_commit_prepare(struct rcu_hash_table *htable)
{
struct rcu_hash_node *node = NULL;
struct rcu_hash_node *tmp = NULL;
@@ -116,23 +164,31 @@ void rcu_hash_update_prepare(struct rcu_hash_table *htable)
void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len, void *data)
{
if (NULL == htable || NULL == key || 0 == key_len) {
return;
}
struct rcu_hash_node *tmp = NULL;
struct rcu_hash_node *node = ALLOC(struct rcu_hash_node, 1);
node->key = (char *)malloc(sizeof(char) * key_len);
memcpy(node->key, key, key_len);
node->key_len = key_len;
node->data = data;
node->htable = htable;
if (!htable->is_updating) {
rcu_hash_update_prepare(htable);
rcu_hash_commit_prepare(htable);
}
if (htable->effective_hash == 'a') {
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, node);
if (NULL == node) {
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, tmp);
if (NULL == tmp) {
HASH_ADD_KEYPTR(hh_b, htable->hashmap_b, key, key_len, node);
}
} else {
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, node);
if (NULL == node) {
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, tmp);
if (NULL == tmp) {
HASH_ADD_KEYPTR(hh_a, htable->hashmap_a, key, key_len, node);
}
}
@@ -140,12 +196,15 @@ void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len
void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len)
{
struct rcu_hash_node *node = NULL;
if (NULL == htable || NULL == key || 0 == key_len) {
return;
}
if (!htable->is_updating) {
rcu_hash_update_prepare(htable);
rcu_hash_commit_prepare(htable);
}
struct rcu_hash_node *node = NULL;
if (htable->effective_hash == 'a') {
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, node);
if (node != NULL) {
@@ -159,14 +218,18 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len
}
if (node != NULL) {
maat_garbage_bagging(htable->garbage_bin, node, (void (*)(void*))rcu_hash_node_free);
rcu_hash_garbage_bagging(&(htable->garbage_q), node, (void (*)(void*))rcu_hash_node_free);
htable->garbage_q_len++;
}
}
void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len)
{
struct rcu_hash_node *node = NULL;
if (NULL == htable || NULL == key || 0 == key_len) {
return NULL;
}
struct rcu_hash_node *node = NULL;
if (htable->effective_hash == 'a') {
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, node);
if (node != NULL) {
@@ -182,8 +245,12 @@ void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_l
return NULL;
}
size_t rcu_hash_counts(struct rcu_hash_table *htable)
size_t rcu_hash_count(struct rcu_hash_table *htable)
{
if (NULL == htable) {
return 0;
}
if (htable->effective_hash == 'a') {
return HASH_CNT(hh_a, htable->hashmap_a);
} else {
@@ -193,6 +260,10 @@ size_t rcu_hash_counts(struct rcu_hash_table *htable)
void rcu_hash_commit(struct rcu_hash_table *htable)
{
if (NULL == htable) {
return;
}
if (!htable->is_updating) {
return;
}
@@ -221,7 +292,40 @@ void rcu_hash_commit(struct rcu_hash_table *htable)
}
}
htable->is_updating = 0;
//maat_garbage_collect_by_force(htable->garbage_bin);
//rcu_garbage
rcu_hash_garbage_queue_free(&(htable->garbage_q));
htable->garbage_q_len = 0;
pthread_mutex_unlock(&htable->update_mutex);
}
size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array)
{
size_t i = 0;
size_t node_cnt = 0;
struct rcu_hash_node *node = NULL, *tmp = NULL;
assert(htable->is_updating == 1);
if (htable->effective_hash == 'a') {
node_cnt = HASH_CNT(hh_b, htable->hashmap_b);
*data_array = ALLOC(void *, node_cnt);
HASH_ITER(hh_b, htable->hashmap_b, node, tmp) {
(*data_array)[i] = node->data;
i++;
}
} else {
node_cnt = HASH_CNT(hh_a, htable->hashmap_a);
*data_array = ALLOC(void *, node_cnt);
HASH_ITER(hh_a, htable->hashmap_a, node, tmp) {
(*data_array)[i] = node->data;
i++;
}
}
return node_cnt;
}
int rcu_hash_updating_flag(struct rcu_hash_table *htable)
{
return htable->is_updating;
}