gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix flag_matcher and interval_matcher compile error

Browse Source
This commit is contained in:
liuwentan
2023-02-07 11:25:31 +08:00
parent 4d2f783874
commit c1902f8deb
27 changed files with 1275 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/CMakeLists.txt
View File

@@ -10,7 +10,7 @@ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp
maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_utils.cpp maat_command.cpp maat_redis_monitor.cpp
maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_intval.cpp maat_flag.cpp
maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_flag.cpp maat_interval.cpp
maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_virtual.cpp)
set(LIB_SOURCE_FILES
@@ -27,7 +27,7 @@ set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX)
set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe)
set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static pthread crypto z)
target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static flagmatcher-static intervalmatcher-static pthread crypto z)
# Shared Library Output
#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})

9
src/inc_internal/maat_expr.h
View File

@@ -42,13 +42,14 @@ int expr_runtime_commit(void *expr_runtime);
* @retval the num of hit group_id
*/
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
int vtable_ids, struct maat_state *state);
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data,
size_t data_len, int result[], size_t *n_result);
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data,
size_t data_len, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
void expr_runtime_stream_close(struct expr_runtime *expr_rt);
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id);

7
src/inc_internal/maat_flag.h
View File

@@ -16,6 +16,8 @@ extern "C"
{
#endif
#include <stdint.h>
#include "log/log.h"
#include "cJSON/cJSON.h"
@@ -42,9 +44,8 @@ int flag_runtime_commit(void *flag_runtime);
* @retval the num of hit group_id
*/
int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
uint64_t flag, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id);
long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread);

57
src/inc_internal/maat_interval.h Normal file
View File

@@ -0,0 +1,57 @@
/*
**********************************************************************************************
* File: maat_interval.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_INTERVAL_H_
#define _MAAT_INTERVAL_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include <stdint.h>
#include "log/log.h"
#include "cJSON/cJSON.h"
struct interval_runtime;
void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void interval_schema_free(void *interval_schema);
/* interval runtime API */
void *interval_runtime_new(void *interval_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void interval_runtime_free(void *interval_runtime);
int interval_runtime_update(void *interval_runtime, void *interval_schema,
const char *line, int valid_column);
int interval_runtime_commit(void *interval_runtime);
/* interval runtime scan API */
/**
* @brief scan intval to get hit group_ids
*
* @retval the num of hit group_id
*/
int interval_runtime_scan_intval(struct interval_runtime *interval_rt, int thread_id,
uint64_t integer, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id);
long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread);
#ifdef __cpluscplus
}
#endif
#endif

56
src/inc_internal/maat_intval.h
View File

@@ -1,56 +0,0 @@
/*
**********************************************************************************************
* File: maat_intval.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_INTVAL_H_
#define _MAAT_INTVAL_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "log/log.h"
#include "cJSON/cJSON.h"
struct intval_runtime;
void *intval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void _schema_free(void *intval_schema);
/* intval runtime API */
void *intval_runtime_new(void *intval_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void intval_runtime_free(void *intval_runtime);
int intval_runtime_update(void *intval_runtime, void *intval_schema,
const char *line, int valid_column);
int intval_runtime_commit(void *intval_runtime);
/* intval runtime scan API */
/**
* @brief scan intval to get hit group_ids
*
* @retval the num of hit group_id
*/
int intval_runtime_scan_intval(struct intval_runtime *intval_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
void intval_runtime_scan_hit_inc(struct intval_runtime *intval_rt, int thread_id);
long long intval_runtime_scan_hit_sum(struct intval_runtime *intval_rt, int n_thread);
#ifdef __cpluscplus
}
#endif
#endif

7
src/inc_internal/maat_rule.h
View File

@@ -31,6 +31,7 @@ extern "C"
#include "IPMatcher.h"
#include "maat_kv.h"
#include "maat_table.h"
#include "maat_virtual.h"
#define MAX_TABLE_NUM 256
@@ -38,6 +39,11 @@ extern "C"
#define MAAT_UPDATE_TYPE_FULL 1
#define MAAT_UPDATE_TYPE_INC 2
#define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2
#define MAX_DISTRICT_STR 128
enum last_scan_flag {
LAST_SCAN_UNSET,
LAST_SCAN_SET,
@@ -253,6 +259,7 @@ struct maat_state {
struct maat_compile_state *compile_state;
};
enum scan_type maat_table_get_scan_type(enum table_type table_type);
size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger);

1
src/inc_internal/maat_table.h
View File

@@ -67,6 +67,7 @@ int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *t
int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str,
size_t district_str_len, int *district_id);
int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district);
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);

2
src/inc_internal/maat_virtual.h
View File

@@ -26,6 +26,7 @@ enum scan_type {
SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING,
SCAN_TYPE_FLAG,
SCAN_TYPE_MAX
};
@@ -34,6 +35,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void virtual_schema_free(void *virtual_schema);
int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type);
#ifdef __cpluscplus
}

134
src/maat_api.cpp
View File

@@ -28,6 +28,7 @@
#include "maat_garbage_collection.h"
#include "maat_group.h"
#include "maat_expr.h"
#include "maat_flag.h"
#include "maat_ip.h"
#include "maat_plugin.h"
#include "maat_ip_plugin.h"
@@ -35,9 +36,6 @@
#define MODULE_MAAT_API module_name_str("maat.api")
#define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2
enum district_set_flag {
DISTRICT_FLAG_UNSET,
DISTRICT_FLAG_SET
@@ -53,17 +51,13 @@ enum scan_type maat_table_get_scan_type(enum table_type table_type)
break;
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
case TABLE_TYPE_SIMILARITY:
case TABLE_TYPE_DIGEST:
ret = SCAN_TYPE_STRING;
break;
case TABLE_TYPE_INTERVAL:
case TABLE_TYPE_INTERVAL_PLUS:
ret = SCAN_TYPE_INTERVAL;
break;
case TABLE_TYPE_IP:
case TABLE_TYPE_IP_PLUS:
case TABLE_TYPE_COMPOSITION:
ret = SCAN_TYPE_IP;
break;
case TABLE_TYPE_PLUGIN:
@@ -84,6 +78,7 @@ enum scan_type maat_table_get_scan_type(enum table_type table_type)
default:
break;
}
return ret;
}
@@ -299,14 +294,6 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
maat_instance->logger = log_handle_create(log_path, opts->log_level);
}
maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags,
maat_instance->garbage_bin, maat_instance->logger);
if (NULL == maat_instance->tbl_mgr) {
goto failed;
}
maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
maat_instance->input_mode = opts->input_mode;
switch (maat_instance->input_mode) {
@@ -343,6 +330,15 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
(maat_instance->gc_timeout_ms / 1000);
maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s);
maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags,
maat_instance->garbage_bin, maat_instance->logger);
if (NULL == maat_instance->tbl_mgr) {
goto failed;
}
maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
maat_instance->outer_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->compile_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
@@ -698,11 +694,97 @@ int hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t compile
}
}
int maat_scan_flag(struct maat *instance, int table_id, int thread_id,
int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
uint64_t flag, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state)
{
return 0;
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM)
|| (thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) {
return MAAT_SCAN_ERR;
}
struct maat_state *mid = NULL;
mid = grab_state(state, maat_instance, thread_id);
mid->scan_cnt++;
int physical_table_id = 0;
int vtable_id = 0;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
//find physical table id
void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_FLAG);
if (physical_table_id < 0) {
return MAAT_SCAN_ERR;
}
} else {
physical_table_id = table_id;
}
if (NULL == maat_instance->maat_rt) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"maat_scan_string error because of maat_runtime is NULL");
return MAAT_SCAN_OK;
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
if (NULL == flag_rt) {
return MAAT_SCAN_ERR;
}
int group_hit_cnt = flag_runtime_scan_flag((struct flag_runtime *)flag_rt,
thread_id,
flag, group_ids,
MAX_SCANNER_HIT_GROUP_NUM,
vtable_id, mid);
if (group_hit_cnt < 0) {
return MAAT_SCAN_ERR;
}
int compile_ret = 0;
if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
if (group_hit_cnt > 0) {
flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id);
}
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;
} else {
compile_table_id = mid->compile_table_id;
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid);
assert(mid->is_last_scan < LAST_SCAN_FINISHED);
if (LAST_SCAN_SET == mid->is_last_scan) {
mid->is_last_scan = LAST_SCAN_FINISHED;
}
}
if (compile_ret > 0) {
alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
}
if (0 == group_hit_cnt && compile_ret > 0) {
// hit NOT group
alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
}
maat_runtime_ref_dec(maat_instance->maat_rt, thread_id);
if (0 == compile_ret && group_hit_cnt > 0) {
return MAAT_SCAN_HALF_HIT;
}
return MAAT_SCAN_HIT;
}
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
@@ -894,11 +976,14 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
mid = grab_state(state, maat_instance, thread_id);
mid->scan_cnt++;
int physical_table_id = -1;
int physical_table_id = 0;
int vtable_id = 0;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
//find physical table id
physical_table_id = virtual_table_get_physical_table_id(table_id, SCAN_TYPE_STRING);
void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_STRING);
if (physical_table_id < 0) {
return MAAT_SCAN_ERR;
}
@@ -914,8 +999,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
//TODO: is TABLE_TYPE_EXPR_PLUS
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
maat_instance->scan_err_cnt++;
@@ -930,7 +1014,8 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
return MAAT_SCAN_ERR;
}
int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, thread_id,
int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt,
thread_id,
data, data_len, group_ids,
MAX_SCANNER_HIT_GROUP_NUM,
vtable_id, mid);
@@ -939,17 +1024,12 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
}
int compile_ret = 0;
int district_id = DISTRICT_ANY;
if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
// come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
}
if (group_hit_cnt > 0 && table_type == TABLE_TYPE_EXPR_PLUS) {
district_id = mid->district_id;
}
int compile_table_id = -1;
if (mid->compile_table_id == -1) {
compile_table_id = maat_instance->default_compile_table_id;

32
src/maat_compile.cpp
View File

@@ -992,7 +992,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
}
//TODO:mytest need to delete
#if 0
#if 1
struct maat_literal_id *p = NULL;
for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) {
printf("compile_id:%d, clause_id:%llu, literal{%d: %d}\n",
@@ -1067,16 +1067,16 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan
size_t ud_result_cnt = 0;
//TODO:mytest need to delete
#if 0
#if 1
unsigned long long *p;
for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clause_array); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clause_array, p))
printf("utarray_len:%u\n", utarray_len(compile_state->all_hit_clauses));
for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clauses); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clauses, p))
{
printf(" before bool_matcher_match compile_state clause_id:%llu\n", *p);
printf("before bool_matcher_match compile_state clause_id:%llu\n", *p);
}
#endif
int bool_match_ret = bool_matcher_match(bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
int bool_match_ret = bool_matcher_match(bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses),
expr_match, MAX_SCANNER_HIT_COMPILE_NUM);
for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) {
@@ -1197,8 +1197,8 @@ struct maat_compile_state *maat_compile_state_new(int thread_id)
compile_state->thread_id = thread_id;
//compile_state->hier_ver = hier->version;
utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd);
utarray_new(compile_state->all_hit_clause_array, &ut_clause_id_icd);
utarray_new(compile_state->this_scan_hit_clause_ids, &ut_clause_id_icd);
utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd);
return compile_state;
}
@@ -1206,8 +1206,8 @@ struct maat_compile_state *maat_compile_state_new(int thread_id)
void maat_compile_state_free(struct maat_compile_state *compile_state)
{
utarray_free(compile_state->internal_hit_paths);
utarray_free(compile_state->all_hit_clause_array);
utarray_free(compile_state->this_scan_hit_clause_ids);
utarray_free(compile_state->all_hit_clauses);
utarray_free(compile_state->this_scan_hit_clauses);
free(compile_state);
}
@@ -1316,8 +1316,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
struct maat_hit_path tmp_path;
struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM;
int bool_match_ret = bool_matcher_match(compile_rt->bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clause_array, 0),
utarray_len(compile_state->all_hit_clause_array), expr_match,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses), expr_match,
MAX_SCANNER_HIT_COMPILE_NUM);
for (int idx = 0; idx < bool_match_ret; idx++) {
compile = (struct maat_compile *)expr_match[idx].user_tag;
@@ -1387,7 +1387,6 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
struct maat_literal_id *tmp = NULL;
unsigned long long *clause_id = 0;
size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) {
for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
@@ -1396,6 +1395,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
continue;
}
size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids,
&literal_id, compare_literal_id);
if (tmp) {
@@ -1404,6 +1404,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
compare_clause_id)) {
continue;
}
utarray_push_back(compile_state->this_scan_hit_clauses, &(clause_state->clause_id));
}
@@ -1779,6 +1780,11 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i],
1, top_group_ids);
if (0 == top_group_cnt) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
group_ids[i], vtable_id);
}
for (int j = 0; j < top_group_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
top_group_ids[j], vtable_id);

93
src/maat_expr.cpp
View File

@@ -23,8 +23,6 @@
#include "alignment.h"
#include "maat_garbage_collection.h"
#define MAX_DISTRICT_STR 128
#define MODULE_EXPR module_name_str("maat.expr")
struct expr_schema {
@@ -161,7 +159,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
}
expr_item->group_id = atoi(line + column_offset);
table_type = table_manager_get_table_type(expr_schema->tbl_mgr, expr_schema->table_id);
table_type = table_manager_get_table_type(expr_schema->ref_tbl_mgr, expr_schema->table_id);
if (table_type == TABLE_TYPE_EXPR_PLUS) {
ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len);
if (ret < 0) {
@@ -339,7 +337,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
read_cnt++;
}
expr_schema->tbl_mgr = tbl_mgr;
expr_schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 8) {
goto error;
@@ -443,7 +441,7 @@ void expr_runtime_free(void *expr_runtime)
}
int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
size_t key_len, and_expr_t *expr_rule,
size_t key_len, int item_id, and_expr_t *expr_rule,
int is_valid, struct log_handle *logger)
{
void *data = NULL;
@@ -454,7 +452,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
if (NULL == data) {
log_error(logger, MODULE_EXPR,
"the key of expr rule not exist, can't be deleted, expr_id:%d",
expr_rule->expr_id);
item_id);
return -1;
}
rcu_hash_del(expr_rt->htable, key, key_len);
@@ -464,7 +462,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
if (data != NULL) {
log_error(logger, MODULE_EXPR,
"the key of expr rule already exist, can't be added, expr_id:%d",
expr_rule->expr_id);
item_id);
return -1;
}
rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule);
@@ -492,7 +490,7 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type)
}
#define MAAT_MAX_EXPR_ITEM_NUM 8
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item,
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data,
struct log_handle *logger)
{
size_t i = 0;
@@ -543,9 +541,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item,
expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]);
expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type);
}
int *district_tag = ALLOC(int, 1);
*district_tag = expr_item->district_id;
expr_rule->user_tag = district_tag;
expr_rule->user_tag = user_data;
expr_rule->n_patterns = sub_expr_cnt;
return expr_rule;
@@ -554,7 +551,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item,
int expr_runtime_update(void *expr_runtime, void *expr_schema,
const char *line, int valid_column)
{
if (NULL == expr_runtime || NULL == expr_schema) {
if (NULL == expr_runtime || NULL == expr_schema ||
NULL == line) {
return -1;
}
@@ -604,7 +602,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
item = maat_item_new(item_id, expr_item->group_id, u_para);
HASH_ADD_INT(expr_rt->item_hash, item_id, item);
expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger);
expr_rule = expr_item_to_expr_rule(expr_item, u_para, expr_rt->logger);
expr_item_free(expr_item);
if (NULL == expr_rule) {
log_error(expr_rt->logger, MODULE_EXPR,
@@ -615,8 +613,8 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
}
char *key = (char *)&item_id;
ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule,
is_valid, expr_rt->logger);
ret = expr_runtime_update_row(expr_rt, key, sizeof(int), item_id,
expr_rule, is_valid, expr_rt->logger);
if (ret < 0) {
if (expr_rule != NULL) {
expr_rule_free(expr_rule);
@@ -653,8 +651,9 @@ int expr_runtime_commit(void *expr_runtime)
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(and_expr_t *)ex_data_array[i];
printf("rule_id:%d\n", rules[i].expr_id);
}
printf("\n\n");
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
@@ -688,15 +687,15 @@ int expr_runtime_commit(void *expr_runtime)
}
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{
size_t n_hit_item = 0;
struct hs_scan_results hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len,
hit_item_ids, MAX_SCANNER_HIT_ITEM_NUM,
hit_results, MAX_SCANNER_HIT_ITEM_NUM,
&n_hit_item);
if (ret < 0) {
return -1;
@@ -713,9 +712,20 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
int district_id = state->district_id;
for (size_t i = 0; i < n_hit_item; i++) {
item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (item->district_id == district_id || district_id == DISTRICT_ANY) {
hit_item_ids[real_hit_item_cnt++] = hit_results[i].item_id;
}
}
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size,
real_hit_item_cnt, group_ids, group_ids_size,
&group_hit_cnt, state);
if (ret < 0) {
return -1;
@@ -735,14 +745,47 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
}
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data,
size_t data_len, int result[], size_t *n_result)
size_t data_len, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{
if (NULL == expr_rt) {
return -1;
}
return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len,
result, n_result);
size_t n_hit_item = 0;
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
int ret = adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM,
&n_hit_item);
if (ret < 0) {
return -1;
}
if (0 == n_hit_item) {
return 0;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(expr_rt->logger, MODULE_EXPR,
"hit expr item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
for (size_t i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].item_id;
}
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size,
&group_hit_cnt, state);
if (ret < 0) {
return -1;
}
return group_hit_cnt;
}
void expr_runtime_stream_close(struct expr_runtime *expr_rt)

395
src/maat_flag.cpp
View File

@@ -8,64 +8,449 @@
***********************************************************************************************
*/
#include <assert.h>
#include "maat_flag.h"
#include "flag_matcher.h"
#include "maat_utils.h"
#include "maat_rule.h"
#include "log/log.h"
#include "uthash/uthash.h"
#include "rcu_hash.h"
#include "maat_table.h"
#include "alignment.h"
#include "maat_compile.h"
#include "maat_garbage_collection.h"
#define MODULE_FLAG module_name_str("maat.flag")
struct flag_schema {
int item_id_column;
int group_id_column;
int flag_column;
int flag_mask_column;
int table_id;
struct table_manager *ref_tbl_mgr;
};
struct flag_item {
int item_id;
int group_id;
uint64_t flag;
uint64_t flag_mask;
};
struct flag_runtime {
struct flag_matcher *matcher;
struct rcu_hash_table *htable;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long *scan_cnt;
long long *hit_cnt;
};
void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
int read_cnt = 0;
struct flag_schema *schema = ALLOC(struct flag_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
schema->table_id = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_FLAG,
"table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->item_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->group_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "flag");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->flag_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "flag_mask");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->flag_mask_column = custom_item->valueint;
read_cnt++;
}
schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 5) {
goto error;
}
return schema;
error:
FREE(schema);
return NULL;
}
void flag_schema_free(void *flag_schema)
{
FREE(flag_schema);
}
void flag_ex_data_free(void *user_ctx, void *data)
{
struct flag_item *flag_item = (struct flag_item *)data;
FREE(flag_item);
}
void *flag_runtime_new(void *flag_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == flag_schema) {
return NULL;
}
struct flag_runtime *flag_rt = ALLOC(struct flag_runtime, 1);
flag_rt->htable = rcu_hash_new(flag_ex_data_free);
flag_rt->item_user_data_free = maat_item_inner_free;
flag_rt->ref_garbage_bin = garbage_bin;
flag_rt->logger = logger;
flag_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
flag_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
return flag_rt;
}
void flag_runtime_free(void *flag_runtime)
{
if (NULL == flag_runtime) {
return;
}
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
if (flag_rt->htable != NULL) {
rcu_hash_free(flag_rt->htable);
flag_rt->htable = NULL;
}
struct maat_item *item = NULL, *tmp = NULL;
HASH_ITER(hh, flag_rt->item_hash, item, tmp) {
HASH_DELETE(hh, flag_rt->item_hash, item);
maat_item_free(item, flag_rt->item_user_data_free);
}
if (flag_rt->hit_cnt != NULL) {
alignment_int64_array_free(flag_rt->hit_cnt);
flag_rt->hit_cnt = NULL;
}
if (flag_rt->scan_cnt != NULL) {
alignment_int64_array_free(flag_rt->scan_cnt);
flag_rt->scan_cnt = NULL;
}
FREE(flag_rt);
}
int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len,
int item_id, struct flag_rule *rule, int is_valid)
{
void *data = NULL;
if (0 == is_valid) {
//delete
data = rcu_hash_find(flag_rt->htable, key, key_len);
if (NULL == data) {
log_error(flag_rt->logger, MODULE_FLAG,
"the key of flag rule not exist, can't be deleted, item_id:%d",
item_id);
return -1;
}
rcu_hash_del(flag_rt->htable, key, key_len);
} else {
//add
data = rcu_hash_find(flag_rt->htable, key, key_len);
if (data != NULL) {
log_error(flag_rt->logger, MODULE_FLAG,
"the key of flag rule already exist, can't be added, item_id:%d",
item_id);
return -1;
}
rcu_hash_add(flag_rt->htable, key, key_len, (void *)rule);
}
return 0;
}
struct flag_item *flag_item_new(const char *line, struct flag_schema *flag_schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
struct flag_item *flag_item = ALLOC(struct flag_item, 1);
int ret = get_column_pos(line, flag_schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no item_id",
flag_schema->table_id, line);
goto error;
}
flag_item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, flag_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no group_id",
flag_schema->table_id, line);
goto error;
}
flag_item->group_id = atoi(line + column_offset);
ret = get_column_pos(line, flag_schema->flag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no flag",
flag_schema->table_id, line);
goto error;
}
flag_item->flag = strtoull(line + column_offset, NULL, 0);
ret = get_column_pos(line, flag_schema->flag_mask_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no flag_mask",
flag_schema->table_id, line);
goto error;
}
flag_item->flag_mask = strtoull(line + column_offset, NULL, 0);
return flag_item;
error:
FREE(flag_item);
return NULL;
}
void flag_item_free(struct flag_item *item)
{
FREE(item);
}
struct flag_rule *flag_item_to_flag_rule(struct flag_item *item, void *user_data)
{
struct flag_rule *rule = ALLOC(struct flag_rule, 1);
rule->rule_id = item->item_id;
rule->flag = item->flag;
rule->mask = item->flag_mask;
rule->user_tag = user_data;
return rule;
}
void flag_rule_free(struct flag_rule *rule)
{
FREE(rule);
}
int flag_runtime_update(void *flag_runtime, void *flag_schema,
const char *line, int valid_column)
{
if (NULL == flag_runtime || NULL == flag_schema ||
NULL == line) {
return -1;
}
struct maat_item_inner *u_para = NULL;
struct maat_item *item = NULL;
struct flag_rule *flag_rule = NULL;
struct flag_schema *schema = (struct flag_schema *)flag_schema;
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND_INT(flag_rt->item_hash, &item_id, item);
if (NULL == item) {
return -1;
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
return -1;
}
HASH_DELETE(hh, flag_rt->item_hash, item);
maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else {
//add
HASH_FIND_INT(flag_rt->item_hash, &item_id, item);
if (item) {
log_error(flag_rt->logger, MODULE_FLAG,
"flag runtime add item %d to item_hash failed, already exist",
item_id);
return -1;
}
struct flag_item *flag_item = flag_item_new(line, schema, flag_rt->logger);
if (NULL == flag_item) {
return -1;
}
u_para = maat_item_inner_new(flag_item->group_id, item_id, 0);
item = maat_item_new(item_id, flag_item->group_id, u_para);
HASH_ADD_INT(flag_rt->item_hash, item_id, item);
flag_rule = flag_item_to_flag_rule(flag_item, u_para);
flag_item_free(flag_item);
if (NULL == flag_rule) {
log_error(flag_rt->logger, MODULE_FLAG,
"transform flag table(table_id:%d) item to flag_rule failed, item_id:%d",
schema->table_id, item_id);
return -1;
}
}
char *key = (char *)&item_id;
int ret = flag_runtime_update_row(flag_rt, key, sizeof(int), item_id, flag_rule, is_valid);
if (ret < 0) {
if (flag_rule != NULL) {
flag_rule_free(flag_rule);
flag_rule = NULL;
}
return -1;
} else {
if (0 == is_valid) {
flag_rt->rule_num--;
} else {
flag_rt->rule_num++;
}
}
return 0;
}
int flag_runtime_commit(void *flag_runtime)
{
if (NULL == flag_runtime) {
return -1;
}
int ret = 0;
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list_updating_data(flag_rt->htable, &ex_data_array);
if (0 == rule_cnt) {
FREE(ex_data_array);
return 0;
}
struct flag_rule *rules = ALLOC(struct flag_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct flag_rule *)ex_data_array[i];
}
struct flag_matcher *new_flag_matcher = NULL;
struct flag_matcher *old_flag_matcher = NULL;
log_info(flag_rt->logger, MODULE_FLAG,
"committing %zu flag rules for rebuilding flag_matcher engine",
rule_cnt);
new_flag_matcher = flag_matcher_new(rules, rule_cnt);
if (NULL == new_flag_matcher) {
log_error(flag_rt->logger, MODULE_FLAG,
"rebuild flag_matcher engine failed when update %zu flag rules",
rule_cnt);
ret = -1;
}
old_flag_matcher = flag_rt->matcher;
flag_rt->matcher = new_flag_matcher;
maat_garbage_bagging(flag_rt->ref_garbage_bin, old_flag_matcher,
(void (*)(void*))flag_matcher_free);
rcu_hash_commit(flag_rt->htable);
rule_cnt = rcu_hash_updating_count(flag_rt->htable);
assert(rule_cnt == 0);
FREE(rules);
FREE(ex_data_array);
return ret;
}
int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
uint64_t flag, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{
struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
int n_hit_item = flag_matcher_match(flag_rt->matcher, flag,
hit_results, MAX_SCANNER_HIT_ITEM_NUM);
if (n_hit_item <= 0) {
return n_hit_item;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(flag_rt->logger, MODULE_FLAG,
"hit flag item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].rule_id;
}
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size,
&group_hit_cnt, state);
if (ret < 0) {
return -1;
}
return group_hit_cnt;
}
void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id)
{
alignment_int64_array_add(flag_rt->hit_cnt, thread_id, 1);
}
long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread)
{
return alignment_int64_array_sum(flag_rt->hit_cnt, n_thread);
}

486
src/maat_interval.cpp Normal file
View File

@@ -0,0 +1,486 @@
/*
**********************************************************************************************
* File: maat_interval.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include <assert.h>
#include "maat_utils.h"
#include "log/log.h"
#include "maat_interval.h"
#include "maat_rule.h"
#include "rcu_hash.h"
#include "alignment.h"
#include "uthash/uthash.h"
#include "maat_garbage_collection.h"
#include "maat_compile.h"
#include "interval_matcher.h"
#define MODULE_INTERVAL module_name_str("maat.interval")
struct interval_schema {
int item_id_column;
int group_id_column;
int district_column;
int low_bound_column;
int up_bound_column;
int table_id;
struct table_manager *ref_tbl_mgr;
};
struct interval_item {
int item_id;
int group_id;
int district_id;
int low_bound;
int up_bound;
};
struct interval_runtime {
struct interval_matcher *matcher;
struct rcu_hash_table *htable;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long *scan_cnt;
long long *hit_cnt;
};
void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
int read_cnt = 0;
struct interval_schema *schema = ALLOC(struct interval_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
schema->table_id = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_INTERVAL,
"table %s has no custom column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->item_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->group_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "district");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->district_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "low_bound");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->low_bound_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "up_bound");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->up_bound_column = custom_item->valueint;
read_cnt++;
}
schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 5) {
goto error;
}
return schema;
error:
FREE(schema);
return NULL;
}
void interval_schema_free(void *interval_schema)
{
FREE(interval_schema);
}
void interval_ex_data_free(void *user_ctx, void *data)
{
struct interval_item *item = (struct interval_item *)data;
FREE(item);
}
void *interval_runtime_new(void *interval_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == interval_schema) {
return NULL;
}
struct interval_runtime *interval_rt = ALLOC(struct interval_runtime, 1);
interval_rt->htable = rcu_hash_new(interval_ex_data_free);
interval_rt->item_user_data_free = maat_item_inner_free;
interval_rt->ref_garbage_bin = garbage_bin;
interval_rt->logger = logger;
interval_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
interval_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
return interval_rt;
}
void interval_runtime_free(void *interval_runtime)
{
if (NULL == interval_runtime) {
return;
}
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
if (interval_rt->htable != NULL) {
rcu_hash_free(interval_rt->htable);
interval_rt->htable = NULL;
}
struct maat_item *item = NULL, *tmp = NULL;
HASH_ITER(hh, interval_rt->item_hash, item, tmp) {
HASH_DELETE(hh, interval_rt->item_hash, item);
maat_item_free(item, interval_rt->item_user_data_free);
}
if (interval_rt->hit_cnt != NULL) {
alignment_int64_array_free(interval_rt->hit_cnt);
interval_rt->hit_cnt = NULL;
}
if (interval_rt->scan_cnt != NULL) {
alignment_int64_array_free(interval_rt->scan_cnt);
interval_rt->scan_cnt = NULL;
}
FREE(interval_rt);
}
struct interval_item *interval_item_new(const char *line, struct interval_schema *schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
enum table_type table_type = TABLE_TYPE_INVALID;
struct interval_item *item = ALLOC(struct interval_item, 1);
int ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s has no item_id",
schema->table_id, line);
goto error;
}
item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s has no group_id",
schema->table_id, line);
goto error;
}
item->group_id = atoi(line + column_offset);
table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id);
if (table_type == TABLE_TYPE_INTERVAL_PLUS) {
ret = get_column_pos(line, schema->district_column, &column_offset, &column_len);
if (ret < 0) {
goto error;
}
if (column_len >= MAX_DISTRICT_STR) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s district length too long",
schema->table_id, line);
goto error;
}
char district[MAX_DISTRICT_STR] = {0};
memcpy(district, (line + column_offset), column_len);
assert(strlen(district) > 0);
str_unescape(district);
item->district_id = table_manager_get_district_id(schema->ref_tbl_mgr, district);
}
ret = get_column_pos(line, schema->low_bound_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s has no low_bound",
schema->table_id, line);
goto error;
}
item->low_bound = atoi(line + column_offset);
ret = get_column_pos(line, schema->up_bound_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s has no up_bound",
schema->table_id, line);
goto error;
}
item->up_bound = atoi(line + column_offset);
return item;
error:
FREE(item);
return NULL;
}
void interval_item_free(void *interval_item)
{
FREE(interval_item);
}
struct interval_rule *interval_item_to_interval_rule(struct interval_item *item, void *user_data)
{
struct interval_rule *rule = ALLOC(struct interval_rule, 1);
rule->start = item->low_bound;
rule->end = item->up_bound;
rule->result.rule_id = item->item_id;
rule->result.user_tag = user_data;
return rule;
}
void interval_rule_free(struct interval_rule *rule)
{
FREE(rule);
}
int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, size_t key_len,
int item_id, struct interval_rule *rule, int is_valid)
{
void *data = NULL;
if (0 == is_valid) {
//delete
data = rcu_hash_find(interval_rt->htable, key, key_len);
if (NULL == data) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"the key of interval rule not exist, can't be deleted, item_id:%d",
item_id);
return -1;
}
rcu_hash_del(interval_rt->htable, key, key_len);
} else {
//add
data = rcu_hash_find(interval_rt->htable, key, key_len);
if (data != NULL) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"the key of interval rule already exist, can't be added, item_id:%d",
item_id);
return -1;
}
rcu_hash_add(interval_rt->htable, key, key_len, (void *)rule);
}
return 0;
}
int interval_runtime_update(void *interval_runtime, void *interval_schema,
const char *line, int valid_column)
{
if (NULL == interval_runtime || NULL == interval_schema ||
NULL == line) {
return -1;
}
struct maat_item_inner *u_para = NULL;
struct maat_item *item = NULL;
struct interval_rule *interval_rule = NULL;
struct interval_schema *schema = (struct interval_schema *)interval_schema;
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
HASH_FIND_INT(interval_rt->item_hash, &item_id, item);
if (NULL == item) {
return -1;
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
return -1;
}
HASH_DELETE(hh, interval_rt->item_hash, item);
maat_garbage_bagging(interval_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else {
//add
HASH_FIND_INT(interval_rt->item_hash, &item_id, item);
if (item) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"interval runtime add item %d to item_hash failed, already exist",
item_id);
return -1;
}
struct interval_item *interval_item = interval_item_new(line, schema, interval_rt->logger);
if (NULL == interval_item) {
return -1;
}
u_para = maat_item_inner_new(interval_item->group_id, item_id, 0);
item = maat_item_new(item_id, interval_item->group_id, u_para);
HASH_ADD_INT(interval_rt->item_hash, item_id, item);
interval_rule = interval_item_to_interval_rule(interval_item, u_para);
interval_item_free(interval_item);
if (NULL == interval_rule) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"transform interval table(table_id:%d) item to interval_rule failed, item_id:%d",
schema->table_id, item_id);
return -1;
}
}
char *key = (char *)&item_id;
int ret = interval_runtime_update_row(interval_rt, key, sizeof(int), item_id, interval_rule, is_valid);
if (ret < 0) {
if (interval_rule != NULL) {
interval_rule_free(interval_rule);
interval_rule = NULL;
}
return -1;
} else {
if (0 == is_valid) {
interval_rt->rule_num--;
} else {
interval_rt->rule_num++;
}
}
return 0;
}
int interval_runtime_commit(void *interval_runtime)
{
if (NULL == interval_runtime) {
return -1;
}
int ret = 0;
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list_updating_data(interval_rt->htable, &ex_data_array);
if (0 == rule_cnt) {
FREE(ex_data_array);
return 0;
}
struct interval_rule *rules = ALLOC(struct interval_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct interval_rule *)ex_data_array[i];
}
struct interval_matcher *new_interval_matcher = NULL;
struct interval_matcher *old_interval_matcher = NULL;
log_info(interval_rt->logger, MODULE_INTERVAL,
"committing %zu interval rules for rebuilding interval_matcher engine",
rule_cnt);
new_interval_matcher = interval_matcher_new(rules, rule_cnt);
if (NULL == new_interval_matcher) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"rebuild interval_matcher engine failed when update %zu interval rules",
rule_cnt);
ret = -1;
}
old_interval_matcher = interval_rt->matcher;
interval_rt->matcher = new_interval_matcher;
maat_garbage_bagging(interval_rt->ref_garbage_bin, old_interval_matcher,
(void (*)(void*))interval_matcher_free);
rcu_hash_commit(interval_rt->htable);
rule_cnt = rcu_hash_updating_count(interval_rt->htable);
assert(rule_cnt == 0);
FREE(rules);
FREE(ex_data_array);
return ret;
}
int interval_runtime_scan_intval(struct interval_runtime *interval_rt, int thread_id,
uint64_t integer,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{
struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
int n_hit_item = interval_matcher_match(interval_rt->matcher, integer,
hit_results, MAX_SCANNER_HIT_ITEM_NUM);
if (n_hit_item <= 0) {
return n_hit_item;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(interval_rt->logger, MODULE_INTERVAL,
"hit interval item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].rule_id;
}
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size,
&group_hit_cnt, state);
if (ret < 0) {
return -1;
}
return group_hit_cnt;
}
void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id)
{
alignment_int64_array_add(interval_rt->hit_cnt, thread_id, 1);
}
long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread)
{
return alignment_int64_array_sum(interval_rt->hit_cnt, n_thread);
}

72
src/maat_intval.cpp
View File

@@ -1,72 +0,0 @@
/*
**********************************************************************************************
* File: maat_intval.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include "maat_intval.h"
struct intval_schema {
};
struct intval_runtime {
};
void *intval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
}
void _schema_free(void *intval_schema)
{
}
/* intval runtime API */
void *intval_runtime_new(void *intval_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
}
void intval_runtime_free(void *intval_runtime)
{
}
int intval_runtime_update(void *intval_runtime, void *intval_schema,
const char *line, int valid_column)
{
}
int intval_runtime_commit(void *intval_runtime)
{
}
int intval_runtime_scan_intval(struct intval_runtime *intval_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{
}
void intval_runtime_scan_hit_inc(struct intval_runtime *intval_rt, int thread_id)
{
}
long long intval_runtime_scan_hit_sum(struct intval_runtime *intval_rt, int n_thread)
{
}

3
src/maat_plugin.cpp
View File

@@ -369,7 +369,8 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *line,
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
const char *line, int valid_column)
{
if (NULL == plugin_runtime || NULL == plugin_schema) {
if (NULL == plugin_runtime || NULL == plugin_schema ||
NULL == line) {
return -1;
}

2
src/maat_rule.cpp
View File

@@ -350,7 +350,7 @@ uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt)
void maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr)
{
size_t table_cnt = table_manager_table_count(tbl_mgr);
enum table_type table_type = TABLE_TYPE_MAX;
enum table_type table_type = TABLE_TYPE_INVALID;
for (size_t i = 0; i < table_cnt; i++) {
table_type = table_manager_get_table_type(tbl_mgr, i);

51
src/maat_table.cpp
View File

@@ -21,6 +21,7 @@
#include "maat_ip.h"
#include "maat_compile.h"
#include "maat_group.h"
#include "maat_flag.h"
#include "maat_plugin.h"
#include "maat_ip_plugin.h"
#include "maat_virtual.h"
@@ -75,14 +76,17 @@ struct table_operations {
};
struct table_operations table_ops[TABLE_TYPE_MAX] = {
// {
// .type = TABLE_TYPE_FLAG,
// .new_schema = flag_schema_new,
// .free_schema = flag_schema_free,
// .new_runtime = flag_runtime_new,
// .free_runtime = flag_runtime_free,
// .update_runtime = flag_runtime_update,
// .commit_runtime = flag_runtime_commit
// },
{
.type = TABLE_TYPE_FLAG,
.new_schema = flag_schema_new,
.free_schema = flag_schema_free,
.new_runtime = flag_runtime_new,
.free_runtime = flag_runtime_free,
.update_runtime = flag_runtime_update,
.commit_runtime = flag_runtime_commit
.type = TABLE_TYPE_FLAG
},
{
.type = TABLE_TYPE_EXPR,
@@ -130,22 +134,10 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_DIGEST,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
.type = TABLE_TYPE_DIGEST
},
{
.type = TABLE_TYPE_SIMILARITY,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
.type = TABLE_TYPE_SIMILARITY
},
{
.type = TABLE_TYPE_CONJUNCTION,
@@ -262,9 +254,13 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map)
maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP);
maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL);
maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS);
maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS);
maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN);
maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN);
maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL);
}
@@ -511,7 +507,7 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n
assert(tbl_mgr->n_table != 0);
size_t i = 0;
enum table_type table_type = TABLE_TYPE_MAX;
enum table_type table_type = TABLE_TYPE_INVALID;
for (i = 0; i < MAX_TABLE_NUM; i++) {
void *schema = table_manager_get_schema(tbl_mgr, i);
@@ -629,11 +625,11 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name)
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id)
{
if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) {
return TABLE_TYPE_MAX;
return TABLE_TYPE_INVALID;
}
if (NULL == tbl_mgr->tbl[table_id]) {
return TABLE_TYPE_MAX;
return TABLE_TYPE_INVALID;
}
return tbl_mgr->tbl[table_id]->table_type;
@@ -750,14 +746,14 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id,
}
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
if (table_type == TABLE_TYPE_MAX) {
if (table_type == TABLE_TYPE_INVALID) {
return -1;
}
if (NULL == table_ops[table_type].update_runtime) {
return -1;
}
int ret = table_ops[table_type].update_runtime(runtime, schema, line, valid_column);
if (tbl_mgr->tmp_district_map != NULL) {
@@ -773,7 +769,7 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id,
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)
{
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
if (table_type == TABLE_TYPE_MAX) {
if (table_type == TABLE_TYPE_INVALID) {
return;
}
@@ -782,6 +778,7 @@ void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)
return;
}
printf("table_id:%d\n", table_id);
if ( table_ops[table_type].commit_runtime != NULL) {
table_ops[table_type].commit_runtime(runtime);
}

46
src/maat_virtual.cpp
View File

@@ -8,11 +8,15 @@
***********************************************************************************************
*/
#include <assert.h>
#include "cJSON/cJSON.h"
#include "maat_kv.h"
#include "maat_utils.h"
#include "log/log.h"
#include "maat_virtual.h"
#include "maat_rule.h"
#include "maat_table.h"
#define MODULE_VIRTUAL module_name_str("maat.virtual")
@@ -25,8 +29,6 @@ struct virtual_schema {
void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
//size_t read_cnt = 0;
cJSON *item = cJSON_GetObjectItem(json, "physical_table");
if (NULL == item || item->type != cJSON_Array) {
log_error(logger, MODULE_VIRTUAL,
@@ -35,37 +37,43 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
}
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
vt_schema->tbl_mgr = tbl_mgr;
return vt_schema;
#if 0
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
int cnt = cJSON_GetArraySize(item);
for (int i = 0; i < cnt; i++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, i);
if (tmp_item != NULL && tmp_item->type == cJSON_String) {
int table_id = -1;
int table_id = table_manager_get_table_id(tbl_mgr, tmp_item->valuestring);
/* physical table should already exist */
int ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &table_id);
if (ret < 0) {
return -1;
}
if (table_id < 0) {
log_error(logger, MODULE_VIRTUAL, "table:%s is not registered",
tmp_item->valuestring);
FREE(vt_schema);
return NULL;
}
enum scan_type table_scan_type = table_schema_get_scan_type(table_array[table_id]);
vt_schema->physical_table_id[table_scan_type]= table_id;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
assert(table_type != TABLE_TYPE_INVALID);
enum scan_type scan_type = maat_table_get_scan_type(table_type);
assert(scan_type != SCAN_TYPE_INVALID);
vt_schema->physical_table_id[scan_type]= table_id;
}
}
if (read_cnt < 3) {
FREE(vt_schema);
return NULL;
}
return vt_schema;
#endif
}
void virtual_schema_free(void *virtual_schema)
{
FREE(virtual_schema);
}
int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type)
{
if (NULL == virtual_schema) {
return -1;
}
struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;
return schema->physical_table_id[type];
}