gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix flag_matcher and interval_matcher compile error

Browse Source
This commit is contained in:
liuwentan
2023-02-07 11:25:31 +08:00
parent 4d2f783874
commit c1902f8deb
27 changed files with 1275 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
scanner/adapter_hs.cpp
View File

@@ -414,7 +414,7 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
size_t n_result, size_t *n_hit_result) size_t n_result, size_t *n_hit_result)
{ {
if (NULL == hs_instance || NULL == data || (0 == data_len) || if (NULL == hs_instance || NULL == data || (0 == data_len) ||
NULL == results || NULL == n_results) { NULL == results || 0 == n_result || NULL == n_hit_result) {
return -1; return -1;
} }
@@ -515,9 +515,8 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
return hs_stream; return hs_stream;
} }
int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len,
const char *data, size_t data_len, struct hs_scan_result *results, size_t n_result, size_t *n_hit_result)
int results[], size_t *n_results)
{ {
hs_error_t err; hs_error_t err;
@@ -549,19 +548,30 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream,
items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->pattern_id_set, i); items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->pattern_id_set, i);
} }
size_t matched_index = 0; int ret = 0;
int matched_index = 0;
struct bool_expr_match *bool_matcher_results = NULL; struct bool_expr_match *bool_matcher_results = NULL;
bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr); bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size, int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size,
bool_matcher_results, hs_stream->n_expr); bool_matcher_results, hs_stream->n_expr);
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) { if (bool_matcher_ret < 0) {
results[matched_index] = bool_matcher_results[matched_index].expr_id; ret = -1;
goto next;
} }
*n_results = bool_matcher_ret;
if (bool_matcher_ret > n_result) {
bool_matcher_ret = n_result;
}
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
results[matched_index].item_id = bool_matcher_results[matched_index].expr_id;
results[matched_index].user_tag = bool_matcher_results[matched_index].user_tag;
}
*n_hit_result = bool_matcher_ret;
next:
FREE(bool_matcher_results); FREE(bool_matcher_results);
return ret;
return 0;
} }
void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream) void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)

2
scanner/adapter_hs.h
View File

@@ -106,7 +106,7 @@ struct adapter_hs_stream;
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id); struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id);
int adapter_hs_scan_stream(struct adapter_hs_stream *stream, const char *data, size_t data_len, int adapter_hs_scan_stream(struct adapter_hs_stream *stream, const char *data, size_t data_len,
int results[], size_t *n_results); struct hs_scan_result *results, size_t n_result, size_t *n_hit_result);
void adapter_hs_stream_close(struct adapter_hs_stream *stream); void adapter_hs_stream_close(struct adapter_hs_stream *stream);

4
src/CMakeLists.txt
View File

@@ -10,7 +10,7 @@ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC) add_definitions(-fPIC)
set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp
maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_utils.cpp maat_command.cpp maat_redis_monitor.cpp maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_utils.cpp maat_command.cpp maat_redis_monitor.cpp
maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_intval.cpp maat_flag.cpp maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_flag.cpp maat_interval.cpp
maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_virtual.cpp) maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_virtual.cpp)
set(LIB_SOURCE_FILES set(LIB_SOURCE_FILES
@@ -27,7 +27,7 @@ set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX)
set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe) set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe)
set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static pthread crypto z) target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static flagmatcher-static intervalmatcher-static pthread crypto z)
# Shared Library Output # Shared Library Output
#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) #add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})

9
src/inc_internal/maat_expr.h
View File

@@ -42,13 +42,14 @@ int expr_runtime_commit(void *expr_runtime);
* @retval the num of hit group_id * @retval the num of hit group_id
*/ */
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len, const char *data, size_t data_len,
int *group_ids, size_t group_ids_size, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state); int vtable_ids, struct maat_state *state);
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data,
size_t data_len, int result[], size_t *n_result); size_t data_len, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
void expr_runtime_stream_close(struct expr_runtime *expr_rt); void expr_runtime_stream_close(struct expr_runtime *expr_rt);
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id); void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id);

7
src/inc_internal/maat_flag.h
View File

@@ -16,6 +16,8 @@ extern "C"
{ {
#endif #endif
#include <stdint.h>
#include "log/log.h" #include "log/log.h"
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
@@ -42,9 +44,8 @@ int flag_runtime_commit(void *flag_runtime);
* @retval the num of hit group_id * @retval the num of hit group_id
*/ */
int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id, int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id,
const char *data, size_t data_len, uint64_t flag, int *group_ids, size_t group_ids_size,
int *group_ids, size_t group_ids_size, int vtable_id, struct maat_state *state);
int vtable_id, struct maat_state *state);
void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id); void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id);
long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread); long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread);

57
src/inc_internal/maat_interval.h Normal file
View File

@@ -0,0 +1,57 @@
/*
**********************************************************************************************
* File: maat_interval.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_INTERVAL_H_
#define _MAAT_INTERVAL_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include <stdint.h>
#include "log/log.h"
#include "cJSON/cJSON.h"
struct interval_runtime;
void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void interval_schema_free(void *interval_schema);
/* interval runtime API */
void *interval_runtime_new(void *interval_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void interval_runtime_free(void *interval_runtime);
int interval_runtime_update(void *interval_runtime, void *interval_schema,
const char *line, int valid_column);
int interval_runtime_commit(void *interval_runtime);
/* interval runtime scan API */
/**
* @brief scan intval to get hit group_ids
*
* @retval the num of hit group_id
*/
int interval_runtime_scan_intval(struct interval_runtime *interval_rt, int thread_id,
uint64_t integer, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id);
long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread);
#ifdef __cpluscplus
}
#endif
#endif

56
src/inc_internal/maat_intval.h
View File

@@ -1,56 +0,0 @@
/*
**********************************************************************************************
* File: maat_intval.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_INTVAL_H_
#define _MAAT_INTVAL_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "log/log.h"
#include "cJSON/cJSON.h"
struct intval_runtime;
void *intval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void _schema_free(void *intval_schema);
/* intval runtime API */
void *intval_runtime_new(void *intval_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void intval_runtime_free(void *intval_runtime);
int intval_runtime_update(void *intval_runtime, void *intval_schema,
const char *line, int valid_column);
int intval_runtime_commit(void *intval_runtime);
/* intval runtime scan API */
/**
* @brief scan intval to get hit group_ids
*
* @retval the num of hit group_id
*/
int intval_runtime_scan_intval(struct intval_runtime *intval_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
void intval_runtime_scan_hit_inc(struct intval_runtime *intval_rt, int thread_id);
long long intval_runtime_scan_hit_sum(struct intval_runtime *intval_rt, int n_thread);
#ifdef __cpluscplus
}
#endif
#endif

7
src/inc_internal/maat_rule.h
View File

@@ -31,6 +31,7 @@ extern "C"
#include "IPMatcher.h" #include "IPMatcher.h"
#include "maat_kv.h" #include "maat_kv.h"
#include "maat_table.h" #include "maat_table.h"
#include "maat_virtual.h"
#define MAX_TABLE_NUM 256 #define MAX_TABLE_NUM 256
@@ -38,6 +39,11 @@ extern "C"
#define MAAT_UPDATE_TYPE_FULL 1 #define MAAT_UPDATE_TYPE_FULL 1
#define MAAT_UPDATE_TYPE_INC 2 #define MAAT_UPDATE_TYPE_INC 2
#define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2
#define MAX_DISTRICT_STR 128
enum last_scan_flag { enum last_scan_flag {
LAST_SCAN_UNSET, LAST_SCAN_UNSET,
LAST_SCAN_SET, LAST_SCAN_SET,
@@ -253,6 +259,7 @@ struct maat_state {
struct maat_compile_state *compile_state; struct maat_compile_state *compile_state;
}; };
enum scan_type maat_table_get_scan_type(enum table_type table_type);
size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger); size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger);

1
src/inc_internal/maat_table.h
View File

@@ -67,6 +67,7 @@ int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *t
int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str, int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str,
size_t district_str_len, int *district_id); size_t district_str_len, int *district_id);
int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district); int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district);
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id); void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id); void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);

2
src/inc_internal/maat_virtual.h
View File

@@ -26,6 +26,7 @@ enum scan_type {
SCAN_TYPE_IP, SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL, SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING, SCAN_TYPE_STRING,
SCAN_TYPE_FLAG,
SCAN_TYPE_MAX SCAN_TYPE_MAX
}; };
@@ -34,6 +35,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void virtual_schema_free(void *virtual_schema); void virtual_schema_free(void *virtual_schema);
int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

134
src/maat_api.cpp
View File

@@ -28,6 +28,7 @@
#include "maat_garbage_collection.h" #include "maat_garbage_collection.h"
#include "maat_group.h" #include "maat_group.h"
#include "maat_expr.h" #include "maat_expr.h"
#include "maat_flag.h"
#include "maat_ip.h" #include "maat_ip.h"
#include "maat_plugin.h" #include "maat_plugin.h"
#include "maat_ip_plugin.h" #include "maat_ip_plugin.h"
@@ -35,9 +36,6 @@
#define MODULE_MAAT_API module_name_str("maat.api") #define MODULE_MAAT_API module_name_str("maat.api")
#define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2
enum district_set_flag { enum district_set_flag {
DISTRICT_FLAG_UNSET, DISTRICT_FLAG_UNSET,
DISTRICT_FLAG_SET DISTRICT_FLAG_SET
@@ -53,17 +51,13 @@ enum scan_type maat_table_get_scan_type(enum table_type table_type)
break; break;
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS: case TABLE_TYPE_EXPR_PLUS:
case TABLE_TYPE_SIMILARITY:
case TABLE_TYPE_DIGEST:
ret = SCAN_TYPE_STRING; ret = SCAN_TYPE_STRING;
break; break;
case TABLE_TYPE_INTERVAL: case TABLE_TYPE_INTERVAL:
case TABLE_TYPE_INTERVAL_PLUS: case TABLE_TYPE_INTERVAL_PLUS:
ret = SCAN_TYPE_INTERVAL; ret = SCAN_TYPE_INTERVAL;
break; break;
case TABLE_TYPE_IP:
case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_IP_PLUS:
case TABLE_TYPE_COMPOSITION:
ret = SCAN_TYPE_IP; ret = SCAN_TYPE_IP;
break; break;
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
@@ -84,6 +78,7 @@ enum scan_type maat_table_get_scan_type(enum table_type table_type)
default: default:
break; break;
} }
return ret; return ret;
} }
@@ -299,14 +294,6 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
maat_instance->logger = log_handle_create(log_path, opts->log_level); maat_instance->logger = log_handle_create(log_path, opts->log_level);
} }
maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags,
maat_instance->garbage_bin, maat_instance->logger);
if (NULL == maat_instance->tbl_mgr) {
goto failed;
}
maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
maat_instance->input_mode = opts->input_mode; maat_instance->input_mode = opts->input_mode;
switch (maat_instance->input_mode) { switch (maat_instance->input_mode) {
@@ -343,6 +330,15 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
(maat_instance->gc_timeout_ms / 1000); (maat_instance->gc_timeout_ms / 1000);
maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s); maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s);
maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags,
maat_instance->garbage_bin, maat_instance->logger);
if (NULL == maat_instance->tbl_mgr) {
goto failed;
}
maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
maat_instance->outer_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); maat_instance->outer_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->compile_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); maat_instance->compile_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
@@ -698,11 +694,97 @@ int hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t compile
} }
} }
int maat_scan_flag(struct maat *instance, int table_id, int thread_id, int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
uint64_t flag, int *results, size_t n_result, uint64_t flag, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state) size_t *n_hit_result, struct maat_state **state)
{ {
return 0; if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM)
|| (thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) {
return MAAT_SCAN_ERR;
}
struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, thread_id);
mid->scan_cnt++;
int physical_table_id = 0;
int vtable_id = 0;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
//find physical table id
void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_FLAG);
if (physical_table_id < 0) {
return MAAT_SCAN_ERR;
}
} else {
physical_table_id = table_id;
}
if (NULL == maat_instance->maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"maat_scan_string error because of maat_runtime is NULL");
return MAAT_SCAN_OK;
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
if (NULL == flag_rt) {
return MAAT_SCAN_ERR;
}
int group_hit_cnt = flag_runtime_scan_flag((struct flag_runtime *)flag_rt,
thread_id,
flag, group_ids,
MAX_SCANNER_HIT_GROUP_NUM,
vtable_id, mid);
if (group_hit_cnt < 0) {
return MAAT_SCAN_ERR;
}
int compile_ret = 0;
if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
if (group_hit_cnt > 0) {
flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id);
}
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;
} else {
compile_table_id = mid->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid);
assert(mid->is_last_scan < LAST_SCAN_FINISHED);
if (LAST_SCAN_SET == mid->is_last_scan) {
mid->is_last_scan = LAST_SCAN_FINISHED;
}
}
if (compile_ret > 0) {
alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
}
if (0 == group_hit_cnt && compile_ret > 0) {
// hit NOT group
alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
}
maat_runtime_ref_dec(maat_instance->maat_rt, thread_id);
if (0 == compile_ret && group_hit_cnt > 0) {
return MAAT_SCAN_HALF_HIT;
}
return MAAT_SCAN_HIT;
} }
int maat_scan_integer(struct maat *instance, int table_id, int thread_id, int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
@@ -894,11 +976,14 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
mid = grab_state(state, maat_instance, thread_id); mid = grab_state(state, maat_instance, thread_id);
mid->scan_cnt++; mid->scan_cnt++;
int physical_table_id = -1; int physical_table_id = 0;
int vtable_id = 0;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) { if (table_type == TABLE_TYPE_VIRTUAL) {
//find physical table id //find physical table id
physical_table_id = virtual_table_get_physical_table_id(table_id, SCAN_TYPE_STRING); void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_STRING);
if (physical_table_id < 0) { if (physical_table_id < 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
@@ -914,8 +999,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
//TODO: is TABLE_TYPE_EXPR_PLUS table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if ((table_type == TABLE_TYPE_EXPR_PLUS) && if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
maat_instance->scan_err_cnt++; maat_instance->scan_err_cnt++;
@@ -930,7 +1014,8 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, thread_id, int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt,
thread_id,
data, data_len, group_ids, data, data_len, group_ids,
MAX_SCANNER_HIT_GROUP_NUM, MAX_SCANNER_HIT_GROUP_NUM,
vtable_id, mid); vtable_id, mid);
@@ -939,17 +1024,12 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
} }
int compile_ret = 0; int compile_ret = 0;
int district_id = DISTRICT_ANY;
if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
if (group_hit_cnt > 0) { if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
} }
if (group_hit_cnt > 0 && table_type == TABLE_TYPE_EXPR_PLUS) {
district_id = mid->district_id;
}
int compile_table_id = -1; int compile_table_id = -1;
if (mid->compile_table_id == -1) { if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id; compile_table_id = maat_instance->default_compile_table_id;

32
src/maat_compile.cpp
View File

@@ -992,7 +992,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
} }
//TODO:mytest need to delete //TODO:mytest need to delete
#if 0 #if 1
struct maat_literal_id *p = NULL; struct maat_literal_id *p = NULL;
for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) { for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) {
printf("compile_id:%d, clause_id:%llu, literal{%d: %d}\n", printf("compile_id:%d, clause_id:%llu, literal{%d: %d}\n",
@@ -1067,16 +1067,16 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan
size_t ud_result_cnt = 0; size_t ud_result_cnt = 0;
//TODO:mytest need to delete //TODO:mytest need to delete
#if 0 #if 1
unsigned long long *p; unsigned long long *p;
for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clause_array); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clause_array, p)) printf("utarray_len:%u\n", utarray_len(compile_state->all_hit_clauses));
for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clauses); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clauses, p))
{ {
printf(" before bool_matcher_match compile_state clause_id:%llu\n", *p); printf("before bool_matcher_match compile_state clause_id:%llu\n", *p);
} }
#endif #endif
int bool_match_ret = bool_matcher_match(bm, int bool_match_ret = bool_matcher_match(bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses), utarray_len(compile_state->all_hit_clauses),
expr_match, MAX_SCANNER_HIT_COMPILE_NUM); expr_match, MAX_SCANNER_HIT_COMPILE_NUM);
for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) { for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) {
@@ -1197,8 +1197,8 @@ struct maat_compile_state *maat_compile_state_new(int thread_id)
compile_state->thread_id = thread_id; compile_state->thread_id = thread_id;
//compile_state->hier_ver = hier->version; //compile_state->hier_ver = hier->version;
utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd); utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd);
utarray_new(compile_state->all_hit_clause_array, &ut_clause_id_icd); utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->this_scan_hit_clause_ids, &ut_clause_id_icd); utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd);
return compile_state; return compile_state;
} }
@@ -1206,8 +1206,8 @@ struct maat_compile_state *maat_compile_state_new(int thread_id)
void maat_compile_state_free(struct maat_compile_state *compile_state) void maat_compile_state_free(struct maat_compile_state *compile_state)
{ {
utarray_free(compile_state->internal_hit_paths); utarray_free(compile_state->internal_hit_paths);
utarray_free(compile_state->all_hit_clause_array); utarray_free(compile_state->all_hit_clauses);
utarray_free(compile_state->this_scan_hit_clause_ids); utarray_free(compile_state->this_scan_hit_clauses);
free(compile_state); free(compile_state);
} }
@@ -1316,8 +1316,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
struct maat_hit_path tmp_path; struct maat_hit_path tmp_path;
struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM; struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM;
int bool_match_ret = bool_matcher_match(compile_rt->bm, int bool_match_ret = bool_matcher_match(compile_rt->bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clause_array, 0), (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clause_array), expr_match, utarray_len(compile_state->all_hit_clauses), expr_match,
MAX_SCANNER_HIT_COMPILE_NUM); MAX_SCANNER_HIT_COMPILE_NUM);
for (int idx = 0; idx < bool_match_ret; idx++) { for (int idx = 0; idx < bool_match_ret; idx++) {
compile = (struct maat_compile *)expr_match[idx].user_tag; compile = (struct maat_compile *)expr_match[idx].user_tag;
@@ -1387,7 +1387,6 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
struct maat_literal_id *tmp = NULL; struct maat_literal_id *tmp = NULL;
unsigned long long *clause_id = 0; unsigned long long *clause_id = 0;
size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) { HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) {
for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
@@ -1396,6 +1395,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
continue; continue;
} }
size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids,
&literal_id, compare_literal_id); &literal_id, compare_literal_id);
if (tmp) { if (tmp) {
@@ -1404,6 +1404,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
compare_clause_id)) { compare_clause_id)) {
continue; continue;
} }
utarray_push_back(compile_state->this_scan_hit_clauses, &(clause_state->clause_id)); utarray_push_back(compile_state->this_scan_hit_clauses, &(clause_state->clause_id));
} }
@@ -1779,6 +1780,11 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i], int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i],
1, top_group_ids); 1, top_group_ids);
if (0 == top_group_cnt) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
group_ids[i], vtable_id);
}
for (int j = 0; j < top_group_cnt; j++) { for (int j = 0; j < top_group_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt, maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
top_group_ids[j], vtable_id); top_group_ids[j], vtable_id);

93
src/maat_expr.cpp
View File

@@ -23,8 +23,6 @@
#include "alignment.h" #include "alignment.h"
#include "maat_garbage_collection.h" #include "maat_garbage_collection.h"
#define MAX_DISTRICT_STR 128
#define MODULE_EXPR module_name_str("maat.expr") #define MODULE_EXPR module_name_str("maat.expr")
struct expr_schema { struct expr_schema {
@@ -161,7 +159,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
} }
expr_item->group_id = atoi(line + column_offset); expr_item->group_id = atoi(line + column_offset);
table_type = table_manager_get_table_type(expr_schema->tbl_mgr, expr_schema->table_id); table_type = table_manager_get_table_type(expr_schema->ref_tbl_mgr, expr_schema->table_id);
if (table_type == TABLE_TYPE_EXPR_PLUS) { if (table_type == TABLE_TYPE_EXPR_PLUS) {
ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -339,7 +337,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
read_cnt++; read_cnt++;
} }
expr_schema->tbl_mgr = tbl_mgr; expr_schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 8) { if (read_cnt < 8) {
goto error; goto error;
@@ -443,7 +441,7 @@ void expr_runtime_free(void *expr_runtime)
} }
int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
size_t key_len, and_expr_t *expr_rule, size_t key_len, int item_id, and_expr_t *expr_rule,
int is_valid, struct log_handle *logger) int is_valid, struct log_handle *logger)
{ {
void *data = NULL; void *data = NULL;
@@ -454,7 +452,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
if (NULL == data) { if (NULL == data) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"the key of expr rule not exist, can't be deleted, expr_id:%d", "the key of expr rule not exist, can't be deleted, expr_id:%d",
expr_rule->expr_id); item_id);
return -1; return -1;
} }
rcu_hash_del(expr_rt->htable, key, key_len); rcu_hash_del(expr_rt->htable, key, key_len);
@@ -464,7 +462,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
if (data != NULL) { if (data != NULL) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"the key of expr rule already exist, can't be added, expr_id:%d", "the key of expr rule already exist, can't be added, expr_id:%d",
expr_rule->expr_id); item_id);
return -1; return -1;
} }
rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule); rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule);
@@ -492,7 +490,7 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type)
} }
#define MAAT_MAX_EXPR_ITEM_NUM 8 #define MAAT_MAX_EXPR_ITEM_NUM 8
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data,
struct log_handle *logger) struct log_handle *logger)
{ {
size_t i = 0; size_t i = 0;
@@ -543,9 +541,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item,
expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]); expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]);
expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type); expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type);
} }
int *district_tag = ALLOC(int, 1);
*district_tag = expr_item->district_id; expr_rule->user_tag = user_data;
expr_rule->user_tag = district_tag;
expr_rule->n_patterns = sub_expr_cnt; expr_rule->n_patterns = sub_expr_cnt;
return expr_rule; return expr_rule;
@@ -554,7 +551,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item,
int expr_runtime_update(void *expr_runtime, void *expr_schema, int expr_runtime_update(void *expr_runtime, void *expr_schema,
const char *line, int valid_column) const char *line, int valid_column)
{ {
if (NULL == expr_runtime || NULL == expr_schema) { if (NULL == expr_runtime || NULL == expr_schema ||
NULL == line) {
return -1; return -1;
} }
@@ -604,7 +602,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
item = maat_item_new(item_id, expr_item->group_id, u_para); item = maat_item_new(item_id, expr_item->group_id, u_para);
HASH_ADD_INT(expr_rt->item_hash, item_id, item); HASH_ADD_INT(expr_rt->item_hash, item_id, item);
expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger); expr_rule = expr_item_to_expr_rule(expr_item, u_para, expr_rt->logger);
expr_item_free(expr_item); expr_item_free(expr_item);
if (NULL == expr_rule) { if (NULL == expr_rule) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
@@ -615,8 +613,8 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
} }
char *key = (char *)&item_id; char *key = (char *)&item_id;
ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, ret = expr_runtime_update_row(expr_rt, key, sizeof(int), item_id,
is_valid, expr_rt->logger); expr_rule, is_valid, expr_rt->logger);
if (ret < 0) { if (ret < 0) {
if (expr_rule != NULL) { if (expr_rule != NULL) {
expr_rule_free(expr_rule); expr_rule_free(expr_rule);
@@ -653,8 +651,9 @@ int expr_runtime_commit(void *expr_runtime)
for (size_t i = 0; i < rule_cnt; i++) { for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(and_expr_t *)ex_data_array[i]; rules[i] = *(and_expr_t *)ex_data_array[i];
printf("rule_id:%d\n", rules[i].expr_id);
} }
printf("\n\n");
struct adapter_hs *new_adapter_hs = NULL; struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL; struct adapter_hs *old_adapter_hs = NULL;
@@ -688,15 +687,15 @@ int expr_runtime_commit(void *expr_runtime)
} }
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len, const char *data, size_t data_len,
int *group_ids, size_t group_ids_size, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state) int vtable_id, struct maat_state *state)
{ {
size_t n_hit_item = 0; size_t n_hit_item = 0;
struct hs_scan_results hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len,
hit_item_ids, MAX_SCANNER_HIT_ITEM_NUM, hit_results, MAX_SCANNER_HIT_ITEM_NUM,
&n_hit_item); &n_hit_item);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
@@ -713,9 +712,20 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
} }
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
int district_id = state->district_id;
for (size_t i = 0; i < n_hit_item; i++) {
item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (item->district_id == district_id || district_id == DISTRICT_ANY) {
hit_item_ids[real_hit_item_cnt++] = hit_results[i].item_id;
}
}
size_t group_hit_cnt = 0; size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size, real_hit_item_cnt, group_ids, group_ids_size,
&group_hit_cnt, state); &group_hit_cnt, state);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
@@ -735,14 +745,47 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
} }
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data,
size_t data_len, int result[], size_t *n_result) size_t data_len, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{ {
if (NULL == expr_rt) { if (NULL == expr_rt) {
return -1; return -1;
} }
return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, size_t n_hit_item = 0;
result, n_result); struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
int ret = adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM,
&n_hit_item);
if (ret < 0) {
return -1;
}
if (0 == n_hit_item) {
return 0;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(expr_rt->logger, MODULE_EXPR,
"hit expr item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
for (size_t i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].item_id;
}
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size,
&group_hit_cnt, state);
if (ret < 0) {
return -1;
}
return group_hit_cnt;
} }
void expr_runtime_stream_close(struct expr_runtime *expr_rt) void expr_runtime_stream_close(struct expr_runtime *expr_rt)

395
src/maat_flag.cpp
View File

@@ -8,64 +8,449 @@
*********************************************************************************************** ***********************************************************************************************
*/ */
#include <assert.h>
#include "maat_flag.h" #include "maat_flag.h"
#include "flag_matcher.h"
#include "maat_utils.h"
#include "maat_rule.h"
#include "log/log.h"
#include "uthash/uthash.h"
#include "rcu_hash.h"
#include "maat_table.h"
#include "alignment.h"
#include "maat_compile.h"
#include "maat_garbage_collection.h"
#define MODULE_FLAG module_name_str("maat.flag")
struct flag_schema { struct flag_schema {
int item_id_column;
int group_id_column;
int flag_column;
int flag_mask_column;
int table_id;
struct table_manager *ref_tbl_mgr;
};
struct flag_item {
int item_id;
int group_id;
uint64_t flag;
uint64_t flag_mask;
}; };
struct flag_runtime { struct flag_runtime {
struct flag_matcher *matcher;
struct rcu_hash_table *htable;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long *scan_cnt;
long long *hit_cnt;
}; };
void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr, void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger) const char *table_name, struct log_handle *logger)
{ {
int read_cnt = 0;
struct flag_schema *schema = ALLOC(struct flag_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
schema->table_id = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_FLAG,
"table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->item_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->group_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "flag");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->flag_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "flag_mask");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->flag_mask_column = custom_item->valueint;
read_cnt++;
}
schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 5) {
goto error;
}
return schema;
error:
FREE(schema);
return NULL;
} }
void flag_schema_free(void *flag_schema) void flag_schema_free(void *flag_schema)
{ {
FREE(flag_schema);
}
void flag_ex_data_free(void *user_ctx, void *data)
{
struct flag_item *flag_item = (struct flag_item *)data;
FREE(flag_item);
} }
void *flag_runtime_new(void *flag_schema, int max_thread_num, void *flag_runtime_new(void *flag_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
if (NULL == flag_schema) {
return NULL;
}
struct flag_runtime *flag_rt = ALLOC(struct flag_runtime, 1);
flag_rt->htable = rcu_hash_new(flag_ex_data_free);
flag_rt->item_user_data_free = maat_item_inner_free;
flag_rt->ref_garbage_bin = garbage_bin;
flag_rt->logger = logger;
flag_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
flag_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
return flag_rt;
} }
void flag_runtime_free(void *flag_runtime) void flag_runtime_free(void *flag_runtime)
{ {
if (NULL == flag_runtime) {
return;
}
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
if (flag_rt->htable != NULL) {
rcu_hash_free(flag_rt->htable);
flag_rt->htable = NULL;
}
struct maat_item *item = NULL, *tmp = NULL;
HASH_ITER(hh, flag_rt->item_hash, item, tmp) {
HASH_DELETE(hh, flag_rt->item_hash, item);
maat_item_free(item, flag_rt->item_user_data_free);
}
if (flag_rt->hit_cnt != NULL) {
alignment_int64_array_free(flag_rt->hit_cnt);
flag_rt->hit_cnt = NULL;
}
if (flag_rt->scan_cnt != NULL) {
alignment_int64_array_free(flag_rt->scan_cnt);
flag_rt->scan_cnt = NULL;
}
FREE(flag_rt);
}
int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len,
int item_id, struct flag_rule *rule, int is_valid)
{
void *data = NULL;
if (0 == is_valid) {
//delete
data = rcu_hash_find(flag_rt->htable, key, key_len);
if (NULL == data) {
log_error(flag_rt->logger, MODULE_FLAG,
"the key of flag rule not exist, can't be deleted, item_id:%d",
item_id);
return -1;
}
rcu_hash_del(flag_rt->htable, key, key_len);
} else {
//add
data = rcu_hash_find(flag_rt->htable, key, key_len);
if (data != NULL) {
log_error(flag_rt->logger, MODULE_FLAG,
"the key of flag rule already exist, can't be added, item_id:%d",
item_id);
return -1;
}
rcu_hash_add(flag_rt->htable, key, key_len, (void *)rule);
}
return 0;
}
struct flag_item *flag_item_new(const char *line, struct flag_schema *flag_schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
struct flag_item *flag_item = ALLOC(struct flag_item, 1);
int ret = get_column_pos(line, flag_schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no item_id",
flag_schema->table_id, line);
goto error;
}
flag_item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, flag_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no group_id",
flag_schema->table_id, line);
goto error;
}
flag_item->group_id = atoi(line + column_offset);
ret = get_column_pos(line, flag_schema->flag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no flag",
flag_schema->table_id, line);
goto error;
}
flag_item->flag = strtoull(line + column_offset, NULL, 0);
ret = get_column_pos(line, flag_schema->flag_mask_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no flag_mask",
flag_schema->table_id, line);
goto error;
}
flag_item->flag_mask = strtoull(line + column_offset, NULL, 0);
return flag_item;
error:
FREE(flag_item);
return NULL;
}
void flag_item_free(struct flag_item *item)
{
FREE(item);
}
struct flag_rule *flag_item_to_flag_rule(struct flag_item *item, void *user_data)
{
struct flag_rule *rule = ALLOC(struct flag_rule, 1);
rule->rule_id = item->item_id;
rule->flag = item->flag;
rule->mask = item->flag_mask;
rule->user_tag = user_data;
return rule;
}
void flag_rule_free(struct flag_rule *rule)
{
FREE(rule);
} }
int flag_runtime_update(void *flag_runtime, void *flag_schema, int flag_runtime_update(void *flag_runtime, void *flag_schema,
const char *line, int valid_column) const char *line, int valid_column)
{ {
if (NULL == flag_runtime || NULL == flag_schema ||
NULL == line) {
return -1;
}
struct maat_item_inner *u_para = NULL;
struct maat_item *item = NULL;
struct flag_rule *flag_rule = NULL;
struct flag_schema *schema = (struct flag_schema *)flag_schema;
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND_INT(flag_rt->item_hash, &item_id, item);
if (NULL == item) {
return -1;
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
return -1;
}
HASH_DELETE(hh, flag_rt->item_hash, item);
maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else {
//add
HASH_FIND_INT(flag_rt->item_hash, &item_id, item);
if (item) {
log_error(flag_rt->logger, MODULE_FLAG,
"flag runtime add item %d to item_hash failed, already exist",
item_id);
return -1;
}
struct flag_item *flag_item = flag_item_new(line, schema, flag_rt->logger);
if (NULL == flag_item) {
return -1;
}
u_para = maat_item_inner_new(flag_item->group_id, item_id, 0);
item = maat_item_new(item_id, flag_item->group_id, u_para);
HASH_ADD_INT(flag_rt->item_hash, item_id, item);
flag_rule = flag_item_to_flag_rule(flag_item, u_para);
flag_item_free(flag_item);
if (NULL == flag_rule) {
log_error(flag_rt->logger, MODULE_FLAG,
"transform flag table(table_id:%d) item to flag_rule failed, item_id:%d",
schema->table_id, item_id);
return -1;
}
}
char *key = (char *)&item_id;
int ret = flag_runtime_update_row(flag_rt, key, sizeof(int), item_id, flag_rule, is_valid);
if (ret < 0) {
if (flag_rule != NULL) {
flag_rule_free(flag_rule);
flag_rule = NULL;
}
return -1;
} else {
if (0 == is_valid) {
flag_rt->rule_num--;
} else {
flag_rt->rule_num++;
}
}
return 0;
} }
int flag_runtime_commit(void *flag_runtime) int flag_runtime_commit(void *flag_runtime)
{ {
if (NULL == flag_runtime) {
return -1;
}
int ret = 0;
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list_updating_data(flag_rt->htable, &ex_data_array);
if (0 == rule_cnt) {
FREE(ex_data_array);
return 0;
}
struct flag_rule *rules = ALLOC(struct flag_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct flag_rule *)ex_data_array[i];
}
struct flag_matcher *new_flag_matcher = NULL;
struct flag_matcher *old_flag_matcher = NULL;
log_info(flag_rt->logger, MODULE_FLAG,
"committing %zu flag rules for rebuilding flag_matcher engine",
rule_cnt);
new_flag_matcher = flag_matcher_new(rules, rule_cnt);
if (NULL == new_flag_matcher) {
log_error(flag_rt->logger, MODULE_FLAG,
"rebuild flag_matcher engine failed when update %zu flag rules",
rule_cnt);
ret = -1;
}
old_flag_matcher = flag_rt->matcher;
flag_rt->matcher = new_flag_matcher;
maat_garbage_bagging(flag_rt->ref_garbage_bin, old_flag_matcher,
(void (*)(void*))flag_matcher_free);
rcu_hash_commit(flag_rt->htable);
rule_cnt = rcu_hash_updating_count(flag_rt->htable);
assert(rule_cnt == 0);
FREE(rules);
FREE(ex_data_array);
return ret;
} }
int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id, int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id,
const char *data, size_t data_len, uint64_t flag, int *group_ids, size_t group_ids_size,
int *group_ids, size_t group_ids_size, int vtable_id, struct maat_state *state)
int vtable_id, struct maat_state *state)
{ {
struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
int n_hit_item = flag_matcher_match(flag_rt->matcher, flag,
hit_results, MAX_SCANNER_HIT_ITEM_NUM);
if (n_hit_item <= 0) {
return n_hit_item;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(flag_rt->logger, MODULE_FLAG,
"hit flag item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].rule_id;
}
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size,
&group_hit_cnt, state);
if (ret < 0) {
return -1;
}
return group_hit_cnt;
} }
void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id) void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id)
{ {
alignment_int64_array_add(flag_rt->hit_cnt, thread_id, 1);
} }
long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread) long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread)
{ {
return alignment_int64_array_sum(flag_rt->hit_cnt, n_thread);
} }

486
src/maat_interval.cpp Normal file
View File

@@ -0,0 +1,486 @@
/*
**********************************************************************************************
* File: maat_interval.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include <assert.h>
#include "maat_utils.h"
#include "log/log.h"
#include "maat_interval.h"
#include "maat_rule.h"
#include "rcu_hash.h"
#include "alignment.h"
#include "uthash/uthash.h"
#include "maat_garbage_collection.h"
#include "maat_compile.h"
#include "interval_matcher.h"
#define MODULE_INTERVAL module_name_str("maat.interval")
struct interval_schema {
int item_id_column;
int group_id_column;
int district_column;
int low_bound_column;
int up_bound_column;
int table_id;
struct table_manager *ref_tbl_mgr;
};
struct interval_item {
int item_id;
int group_id;
int district_id;
int low_bound;
int up_bound;
};
struct interval_runtime {
struct interval_matcher *matcher;
struct rcu_hash_table *htable;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long *scan_cnt;
long long *hit_cnt;
};
void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
int read_cnt = 0;
struct interval_schema *schema = ALLOC(struct interval_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
schema->table_id = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_INTERVAL,
"table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->item_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->group_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "district");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->district_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "low_bound");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->low_bound_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "up_bound");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->up_bound_column = custom_item->valueint;
read_cnt++;
}
schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 5) {
goto error;
}
return schema;
error:
FREE(schema);
return NULL;
}
void interval_schema_free(void *interval_schema)
{
FREE(interval_schema);
}
void interval_ex_data_free(void *user_ctx, void *data)
{
struct interval_item *item = (struct interval_item *)data;
FREE(item);
}
void *interval_runtime_new(void *interval_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == interval_schema) {
return NULL;
}
struct interval_runtime *interval_rt = ALLOC(struct interval_runtime, 1);
interval_rt->htable = rcu_hash_new(interval_ex_data_free);
interval_rt->item_user_data_free = maat_item_inner_free;
interval_rt->ref_garbage_bin = garbage_bin;
interval_rt->logger = logger;
interval_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
interval_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
return interval_rt;
}
void interval_runtime_free(void *interval_runtime)
{
if (NULL == interval_runtime) {
return;
}
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
if (interval_rt->htable != NULL) {
rcu_hash_free(interval_rt->htable);
interval_rt->htable = NULL;
}
struct maat_item *item = NULL, *tmp = NULL;
HASH_ITER(hh, interval_rt->item_hash, item, tmp) {
HASH_DELETE(hh, interval_rt->item_hash, item);
maat_item_free(item, interval_rt->item_user_data_free);
}
if (interval_rt->hit_cnt != NULL) {
alignment_int64_array_free(interval_rt->hit_cnt);
interval_rt->hit_cnt = NULL;
}
if (interval_rt->scan_cnt != NULL) {
alignment_int64_array_free(interval_rt->scan_cnt);
interval_rt->scan_cnt = NULL;
}
FREE(interval_rt);
}
struct interval_item *interval_item_new(const char *line, struct interval_schema *schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
enum table_type table_type = TABLE_TYPE_INVALID;
struct interval_item *item = ALLOC(struct interval_item, 1);
int ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s has no item_id",
schema->table_id, line);
goto error;
}
item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s has no group_id",
schema->table_id, line);
goto error;
}
item->group_id = atoi(line + column_offset);
table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id);
if (table_type == TABLE_TYPE_INTERVAL_PLUS) {
ret = get_column_pos(line, schema->district_column, &column_offset, &column_len);
if (ret < 0) {
goto error;
}
if (column_len >= MAX_DISTRICT_STR) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s district length too long",
schema->table_id, line);
goto error;
}
char district[MAX_DISTRICT_STR] = {0};
memcpy(district, (line + column_offset), column_len);
assert(strlen(district) > 0);
str_unescape(district);
item->district_id = table_manager_get_district_id(schema->ref_tbl_mgr, district);
}
ret = get_column_pos(line, schema->low_bound_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s has no low_bound",
schema->table_id, line);
goto error;
}
item->low_bound = atoi(line + column_offset);
ret = get_column_pos(line, schema->up_bound_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s has no up_bound",
schema->table_id, line);
goto error;
}
item->up_bound = atoi(line + column_offset);
return item;
error:
FREE(item);
return NULL;
}
void interval_item_free(void *interval_item)
{
FREE(interval_item);
}
struct interval_rule *interval_item_to_interval_rule(struct interval_item *item, void *user_data)
{
struct interval_rule *rule = ALLOC(struct interval_rule, 1);
rule->start = item->low_bound;
rule->end = item->up_bound;
rule->result.rule_id = item->item_id;
rule->result.user_tag = user_data;
return rule;
}
void interval_rule_free(struct interval_rule *rule)
{
FREE(rule);
}
int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, size_t key_len,
int item_id, struct interval_rule *rule, int is_valid)
{
void *data = NULL;
if (0 == is_valid) {
//delete
data = rcu_hash_find(interval_rt->htable, key, key_len);
if (NULL == data) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"the key of interval rule not exist, can't be deleted, item_id:%d",
item_id);
return -1;
}
rcu_hash_del(interval_rt->htable, key, key_len);
} else {
//add
data = rcu_hash_find(interval_rt->htable, key, key_len);
if (data != NULL) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"the key of interval rule already exist, can't be added, item_id:%d",
item_id);
return -1;
}
rcu_hash_add(interval_rt->htable, key, key_len, (void *)rule);
}
return 0;
}
int interval_runtime_update(void *interval_runtime, void *interval_schema,
const char *line, int valid_column)
{
if (NULL == interval_runtime || NULL == interval_schema ||
NULL == line) {
return -1;
}
struct maat_item_inner *u_para = NULL;
struct maat_item *item = NULL;
struct interval_rule *interval_rule = NULL;
struct interval_schema *schema = (struct interval_schema *)interval_schema;
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND_INT(interval_rt->item_hash, &item_id, item);
if (NULL == item) {
return -1;
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
return -1;
}
HASH_DELETE(hh, interval_rt->item_hash, item);
maat_garbage_bagging(interval_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else {
//add
HASH_FIND_INT(interval_rt->item_hash, &item_id, item);
if (item) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"interval runtime add item %d to item_hash failed, already exist",
item_id);
return -1;
}
struct interval_item *interval_item = interval_item_new(line, schema, interval_rt->logger);
if (NULL == interval_item) {
return -1;
}
u_para = maat_item_inner_new(interval_item->group_id, item_id, 0);
item = maat_item_new(item_id, interval_item->group_id, u_para);
HASH_ADD_INT(interval_rt->item_hash, item_id, item);
interval_rule = interval_item_to_interval_rule(interval_item, u_para);
interval_item_free(interval_item);
if (NULL == interval_rule) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"transform interval table(table_id:%d) item to interval_rule failed, item_id:%d",
schema->table_id, item_id);
return -1;
}
}
char *key = (char *)&item_id;
int ret = interval_runtime_update_row(interval_rt, key, sizeof(int), item_id, interval_rule, is_valid);
if (ret < 0) {
if (interval_rule != NULL) {
interval_rule_free(interval_rule);
interval_rule = NULL;
}
return -1;
} else {
if (0 == is_valid) {
interval_rt->rule_num--;
} else {
interval_rt->rule_num++;
}
}
return 0;
}
int interval_runtime_commit(void *interval_runtime)
{
if (NULL == interval_runtime) {
return -1;
}
int ret = 0;
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list_updating_data(interval_rt->htable, &ex_data_array);
if (0 == rule_cnt) {
FREE(ex_data_array);
return 0;
}
struct interval_rule *rules = ALLOC(struct interval_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct interval_rule *)ex_data_array[i];
}
struct interval_matcher *new_interval_matcher = NULL;
struct interval_matcher *old_interval_matcher = NULL;
log_info(interval_rt->logger, MODULE_INTERVAL,
"committing %zu interval rules for rebuilding interval_matcher engine",
rule_cnt);
new_interval_matcher = interval_matcher_new(rules, rule_cnt);
if (NULL == new_interval_matcher) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"rebuild interval_matcher engine failed when update %zu interval rules",
rule_cnt);
ret = -1;
}
old_interval_matcher = interval_rt->matcher;
interval_rt->matcher = new_interval_matcher;
maat_garbage_bagging(interval_rt->ref_garbage_bin, old_interval_matcher,
(void (*)(void*))interval_matcher_free);
rcu_hash_commit(interval_rt->htable);
rule_cnt = rcu_hash_updating_count(interval_rt->htable);
assert(rule_cnt == 0);
FREE(rules);
FREE(ex_data_array);
return ret;
}
int interval_runtime_scan_intval(struct interval_runtime *interval_rt, int thread_id,
uint64_t integer,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{
struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
int n_hit_item = interval_matcher_match(interval_rt->matcher, integer,
hit_results, MAX_SCANNER_HIT_ITEM_NUM);
if (n_hit_item <= 0) {
return n_hit_item;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(interval_rt->logger, MODULE_INTERVAL,
"hit interval item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].rule_id;
}
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size,
&group_hit_cnt, state);
if (ret < 0) {
return -1;
}
return group_hit_cnt;
}
void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id)
{
alignment_int64_array_add(interval_rt->hit_cnt, thread_id, 1);
}
long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread)
{
return alignment_int64_array_sum(interval_rt->hit_cnt, n_thread);
}

72
src/maat_intval.cpp
View File

@@ -1,72 +0,0 @@
/*
**********************************************************************************************
* File: maat_intval.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include "maat_intval.h"
struct intval_schema {
};
struct intval_runtime {
};
void *intval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
}
void _schema_free(void *intval_schema)
{
}
/* intval runtime API */
void *intval_runtime_new(void *intval_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
}
void intval_runtime_free(void *intval_runtime)
{
}
int intval_runtime_update(void *intval_runtime, void *intval_schema,
const char *line, int valid_column)
{
}
int intval_runtime_commit(void *intval_runtime)
{
}
int intval_runtime_scan_intval(struct intval_runtime *intval_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{
}
void intval_runtime_scan_hit_inc(struct intval_runtime *intval_rt, int thread_id)
{
}
long long intval_runtime_scan_hit_sum(struct intval_runtime *intval_rt, int n_thread)
{
}

3
src/maat_plugin.cpp
View File

@@ -369,7 +369,8 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *line,
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
const char *line, int valid_column) const char *line, int valid_column)
{ {
if (NULL == plugin_runtime || NULL == plugin_schema) { if (NULL == plugin_runtime || NULL == plugin_schema ||
NULL == line) {
return -1; return -1;
} }

2
src/maat_rule.cpp
View File

@@ -350,7 +350,7 @@ uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt)
void maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr) void maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr)
{ {
size_t table_cnt = table_manager_table_count(tbl_mgr); size_t table_cnt = table_manager_table_count(tbl_mgr);
enum table_type table_type = TABLE_TYPE_MAX; enum table_type table_type = TABLE_TYPE_INVALID;
for (size_t i = 0; i < table_cnt; i++) { for (size_t i = 0; i < table_cnt; i++) {
table_type = table_manager_get_table_type(tbl_mgr, i); table_type = table_manager_get_table_type(tbl_mgr, i);

51
src/maat_table.cpp
View File

@@ -21,6 +21,7 @@
#include "maat_ip.h" #include "maat_ip.h"
#include "maat_compile.h" #include "maat_compile.h"
#include "maat_group.h" #include "maat_group.h"
#include "maat_flag.h"
#include "maat_plugin.h" #include "maat_plugin.h"
#include "maat_ip_plugin.h" #include "maat_ip_plugin.h"
#include "maat_virtual.h" #include "maat_virtual.h"
@@ -75,14 +76,17 @@ struct table_operations {
}; };
struct table_operations table_ops[TABLE_TYPE_MAX] = { struct table_operations table_ops[TABLE_TYPE_MAX] = {
// {
// .type = TABLE_TYPE_FLAG,
// .new_schema = flag_schema_new,
// .free_schema = flag_schema_free,
// .new_runtime = flag_runtime_new,
// .free_runtime = flag_runtime_free,
// .update_runtime = flag_runtime_update,
// .commit_runtime = flag_runtime_commit
// },
{ {
.type = TABLE_TYPE_FLAG, .type = TABLE_TYPE_FLAG
.new_schema = flag_schema_new,
.free_schema = flag_schema_free,
.new_runtime = flag_runtime_new,
.free_runtime = flag_runtime_free,
.update_runtime = flag_runtime_update,
.commit_runtime = flag_runtime_commit
}, },
{ {
.type = TABLE_TYPE_EXPR, .type = TABLE_TYPE_EXPR,
@@ -130,22 +134,10 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.commit_runtime = NULL .commit_runtime = NULL
}, },
{ {
.type = TABLE_TYPE_DIGEST, .type = TABLE_TYPE_DIGEST
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
}, },
{ {
.type = TABLE_TYPE_SIMILARITY, .type = TABLE_TYPE_SIMILARITY
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
}, },
{ {
.type = TABLE_TYPE_CONJUNCTION, .type = TABLE_TYPE_CONJUNCTION,
@@ -262,9 +254,13 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map)
maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP);
maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL);
maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS);
maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS);
maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN);
maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN);
maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL);
} }
@@ -511,7 +507,7 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n
assert(tbl_mgr->n_table != 0); assert(tbl_mgr->n_table != 0);
size_t i = 0; size_t i = 0;
enum table_type table_type = TABLE_TYPE_MAX; enum table_type table_type = TABLE_TYPE_INVALID;
for (i = 0; i < MAX_TABLE_NUM; i++) { for (i = 0; i < MAX_TABLE_NUM; i++) {
void *schema = table_manager_get_schema(tbl_mgr, i); void *schema = table_manager_get_schema(tbl_mgr, i);
@@ -629,11 +625,11 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name)
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id) enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id)
{ {
if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) { if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) {
return TABLE_TYPE_MAX; return TABLE_TYPE_INVALID;
} }
if (NULL == tbl_mgr->tbl[table_id]) { if (NULL == tbl_mgr->tbl[table_id]) {
return TABLE_TYPE_MAX; return TABLE_TYPE_INVALID;
} }
return tbl_mgr->tbl[table_id]->table_type; return tbl_mgr->tbl[table_id]->table_type;
@@ -750,14 +746,14 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id,
} }
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
if (table_type == TABLE_TYPE_MAX) { if (table_type == TABLE_TYPE_INVALID) {
return -1; return -1;
} }
if (NULL == table_ops[table_type].update_runtime) { if (NULL == table_ops[table_type].update_runtime) {
return -1; return -1;
} }
int ret = table_ops[table_type].update_runtime(runtime, schema, line, valid_column); int ret = table_ops[table_type].update_runtime(runtime, schema, line, valid_column);
if (tbl_mgr->tmp_district_map != NULL) { if (tbl_mgr->tmp_district_map != NULL) {
@@ -773,7 +769,7 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id,
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id) void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)
{ {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
if (table_type == TABLE_TYPE_MAX) { if (table_type == TABLE_TYPE_INVALID) {
return; return;
} }
@@ -782,6 +778,7 @@ void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)
return; return;
} }
printf("table_id:%d\n", table_id);
if ( table_ops[table_type].commit_runtime != NULL) { if ( table_ops[table_type].commit_runtime != NULL) {
table_ops[table_type].commit_runtime(runtime); table_ops[table_type].commit_runtime(runtime);
} }

46
src/maat_virtual.cpp
View File

@@ -8,11 +8,15 @@
*********************************************************************************************** ***********************************************************************************************
*/ */
#include <assert.h>
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
#include "maat_kv.h" #include "maat_kv.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "log/log.h" #include "log/log.h"
#include "maat_virtual.h" #include "maat_virtual.h"
#include "maat_rule.h"
#include "maat_table.h"
#define MODULE_VIRTUAL module_name_str("maat.virtual") #define MODULE_VIRTUAL module_name_str("maat.virtual")
@@ -25,8 +29,6 @@ struct virtual_schema {
void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger) const char *table_name, struct log_handle *logger)
{ {
//size_t read_cnt = 0;
cJSON *item = cJSON_GetObjectItem(json, "physical_table"); cJSON *item = cJSON_GetObjectItem(json, "physical_table");
if (NULL == item || item->type != cJSON_Array) { if (NULL == item || item->type != cJSON_Array) {
log_error(logger, MODULE_VIRTUAL, log_error(logger, MODULE_VIRTUAL,
@@ -35,37 +37,43 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
} }
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
vt_schema->tbl_mgr = tbl_mgr; vt_schema->tbl_mgr = tbl_mgr;
return vt_schema;
#if 0
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
int cnt = cJSON_GetArraySize(item); int cnt = cJSON_GetArraySize(item);
for (int i = 0; i < cnt; i++) { for (int i = 0; i < cnt; i++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, i); cJSON *tmp_item = cJSON_GetArrayItem(item, i);
if (tmp_item != NULL && tmp_item->type == cJSON_String) { if (tmp_item != NULL && tmp_item->type == cJSON_String) {
int table_id = -1; int table_id = table_manager_get_table_id(tbl_mgr, tmp_item->valuestring);
/* physical table should already exist */ /* physical table should already exist */
int ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &table_id); if (table_id < 0) {
if (ret < 0) { log_error(logger, MODULE_VIRTUAL, "table:%s is not registered",
return -1; tmp_item->valuestring);
} FREE(vt_schema);
return NULL;
}
enum scan_type table_scan_type = table_schema_get_scan_type(table_array[table_id]); enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
vt_schema->physical_table_id[table_scan_type]= table_id; assert(table_type != TABLE_TYPE_INVALID);
enum scan_type scan_type = maat_table_get_scan_type(table_type);
assert(scan_type != SCAN_TYPE_INVALID);
vt_schema->physical_table_id[scan_type]= table_id;
} }
} }
if (read_cnt < 3) {
FREE(vt_schema);
return NULL;
}
return vt_schema; return vt_schema;
#endif
} }
void virtual_schema_free(void *virtual_schema) void virtual_schema_free(void *virtual_schema)
{ {
FREE(virtual_schema); FREE(virtual_schema);
}
int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type)
{
if (NULL == virtual_schema) {
return -1;
}
struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;
return schema->physical_table_id[type];
} }

16
test/maat_framework_gtest.cpp
View File

@@ -113,7 +113,7 @@ TEST_F(MaatIPScan, hit_ip_and_port) {
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
@@ -148,7 +148,7 @@ TEST_F(MaatStringScan, hit_ip_and_port_range) {
n_hit_result = 0; n_hit_result = 0;
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
} }
@@ -183,7 +183,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
} }
@@ -209,7 +209,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) {
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
} }
@@ -233,7 +233,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
state = NULL; state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
} }
@@ -256,7 +256,7 @@ TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) {
state = NULL; state = NULL;
ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
} }
@@ -269,7 +269,7 @@ TEST(maat_scan_string, dynamic_config) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
@@ -303,7 +303,7 @@ TEST(maat_scan_ip, dynamic_config) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);

25
test/maat_input_mode_gtest.cpp
View File

@@ -10,6 +10,7 @@
const char *table_info_path = "./table_info.conf"; const char *table_info_path = "./table_info.conf";
const char *json_filename = "maat_json.json"; const char *json_filename = "maat_json.json";
#if 0
TEST(json_mode, maat_scan_string) { TEST(json_mode, maat_scan_string) {
char json_iris_path[128] = {0}; char json_iris_path[128] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
@@ -26,16 +27,16 @@ TEST(json_mode, maat_scan_string) {
struct maat *maat_instance = maat_new(opts, table_info_path); struct maat *maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(maat_instance, nullptr); EXPECT_NE(maat_instance, nullptr);
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL"); int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE");
char scan_data[128] = "hello"; char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
int results[5] = {0}; int results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191); EXPECT_EQ(results[0], 132);
maat_options_free(opts); maat_options_free(opts);
maat_state_free(&state); maat_state_free(&state);
@@ -69,22 +70,22 @@ TEST(iris_mode, maat_scan_string) {
struct maat *maat_instance = maat_new(opts, table_info_path); struct maat *maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(maat_instance, nullptr); EXPECT_NE(maat_instance, nullptr);
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL"); int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE");
char scan_data[128] = "hello"; char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
int results[5] = {0}; int results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191); EXPECT_EQ(results[0], 132);
maat_options_free(opts); maat_options_free(opts);
maat_state_free(&state); maat_state_free(&state);
maat_free(maat_instance); maat_free(maat_instance);
} }
#endif
int count_line_num_cb(const char *table_name, const char *line, void *u_para) int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{ {
(*((unsigned int *)u_para))++; (*((unsigned int *)u_para))++;
@@ -131,6 +132,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
return 0; return 0;
} }
#if 1
TEST(redis_mode, maat_scan_string) { TEST(redis_mode, maat_scan_string) {
char json_iris_path[128] = {0}; char json_iris_path[128] = {0};
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
@@ -189,23 +192,23 @@ TEST(redis_mode, maat_scan_string) {
maat_options_set_logger(opts, logger); maat_options_set_logger(opts, logger);
struct maat *maat_instance = maat_new(opts, table_info_path); struct maat *maat_instance = maat_new(opts, table_info_path);
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL"); int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE");
char scan_data[128] = "hello"; char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
int results[5] = {0}; int results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191); EXPECT_EQ(results[0], 132);
maat_options_free(opts); maat_options_free(opts);
maat_state_free(&state); maat_state_free(&state);
log_handle_destroy(maat_instance->logger); log_handle_destroy(maat_instance->logger);
maat_free(maat_instance); maat_free(maat_instance);
} }
#endif
int main(int argc, char ** argv) int main(int argc, char ** argv)
{ {
int ret=0; int ret=0;

10
test/table_info.conf
View File

@@ -20,7 +20,7 @@
"table_id":1, "table_id":1,
"table_name":"GROUP2COMPILE", "table_name":"GROUP2COMPILE",
"table_type":"group2compile", "table_type":"group2compile",
"associated_compile_table_id":1, "associated_compile_table_id":0,
"valid_column":3, "valid_column":3,
"custom": { "custom": {
"group_id":1, "group_id":1,
@@ -113,6 +113,7 @@
"table_type":"expr_plus", "table_type":"expr_plus",
"valid_column":8, "valid_column":8,
"custom": { "custom": {
"scan_mode":"block",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"district":3, "district":3,
@@ -210,7 +211,6 @@
"expr_type":5, "expr_type":5,
"match_method":6, "match_method":6,
"is_hexbin":7 "is_hexbin":7
} }
}, },
{ {
@@ -377,12 +377,6 @@
"fqdn":3 "fqdn":3
} }
}, },
{
"table_id":37,
"table_name":"VIRTUAL_SSL_SNI",
"table_type":"virtual",
"physical_table": ["KEYWORDS_TABLE", "INTERGER_PLUS"]
},
{ {
"table_id":38, "table_id":38,
"table_name":"APP_ID", "table_name":"APP_ID",

28
vendor/CMakeLists.txt vendored
View File

@@ -99,4 +99,30 @@ file(MAKE_DIRECTORY ${INSTALL_DIR}/include)
add_library(igraph-static STATIC IMPORTED GLOBAL) add_library(igraph-static STATIC IMPORTED GLOBAL)
add_dependencies(igraph-static igraph) add_dependencies(igraph-static igraph)
set_property(TARGET igraph-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libigraph.a) set_property(TARGET igraph-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libigraph.a)
set_property(TARGET igraph-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) set_property(TARGET igraph-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include)
#flag-matcher
ExternalProject_Add(flagmatcher PREFIX flagmatcher
URL ${CMAKE_CURRENT_SOURCE_DIR}/flag-matcher.tar.gz
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC")
ExternalProject_Get_Property(flagmatcher INSTALL_DIR)
file(MAKE_DIRECTORY ${VENDOR_BUILD}/include)
add_library(flagmatcher-static STATIC IMPORTED GLOBAL)
add_dependencies(flagmatcher-static flagmatcher)
set_property(TARGET flagmatcher-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib/libflag_matcher.a)
set_property(TARGET flagmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include)
#interval-matcher
ExternalProject_Add(intervalmatcher PREFIX intervalmatcher
URL ${CMAKE_CURRENT_SOURCE_DIR}/interval-matcher.tar.gz
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC")
ExternalProject_Get_Property(intervalmatcher INSTALL_DIR)
file(MAKE_DIRECTORY ${VENDOR_BUILD}/include)
add_library(intervalmatcher-static STATIC IMPORTED GLOBAL)
add_dependencies(intervalmatcher-static flagmatcher)
set_property(TARGET intervalmatcher-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib/libinterval_matcher.a)
set_property(TARGET intervalmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include)

BIN
vendor/flag_matcher.tar.gz → vendor/flag-matcher.tar.gz vendored
View File

Binary file not shown.

BIN
vendor/interval-matcher.tar.gz vendored Normal file
View File

Binary file not shown.

BIN
vendor/interval_matcher.tar.gz vendored
View File

Binary file not shown.