gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compile_id,group_id,item_id support uint64_t

Browse Source
This commit is contained in:
liuwentan
2023-02-22 15:08:52 +08:00
parent 313b0558d0
commit ac51c70426
30 changed files with 948 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
include/maat.h
View File

@@ -29,16 +29,16 @@ struct maat;
struct maat_hit_path { struct maat_hit_path {
int Nth_scan; int Nth_scan;
int item_id; int vtable_id; // 0 is not a virtual table.
int sub_group_id; uint64_t item_id;
int top_group_id; uint64_t sub_group_id;
int vtable_id; // 0 is not a virtual table. uint64_t top_group_id;
int compile_id; uint64_t compile_id;
}; };
struct maat_hit_object { struct maat_hit_object {
int vtable_id; int vtable_id;
int group_id; uint64_t group_id;
}; };
enum maat_scan_status { enum maat_scan_status {
@@ -50,7 +50,7 @@ enum maat_scan_status {
#define MAX_SERVICE_DEFINE_LEN 128 #define MAX_SERVICE_DEFINE_LEN 128
struct maat_rule { struct maat_rule {
int config_id; uint64_t config_id;
uint8_t reserved; uint8_t reserved;
int serv_def_len; int serv_def_len;
char service_defined[MAX_SERVICE_DEFINE_LEN]; char service_defined[MAX_SERVICE_DEFINE_LEN];
@@ -116,7 +116,7 @@ int maat_compile_table_ex_schema_register(struct maat *instance, int table_id,
maat_rule_ex_dup_func_t *dup_func, maat_rule_ex_dup_func_t *dup_func,
long argl, void *argp); long argl, void *argp);
void *maat_compile_table_get_ex_data(struct maat *instance, int compile_table_id, void *maat_compile_table_get_ex_data(struct maat *instance, int compile_table_id,
int compile_id, size_t idx); uint64_t compile_id, size_t idx);
/* maat plugin table API */ /* maat plugin table API */
int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id, int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id,
@@ -126,6 +126,8 @@ int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id,
long argl, void *argp); long argl, void *argp);
/* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register, /* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register,
caller is responsible to free the data. */ caller is responsible to free the data. */
void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id, const char *key);
int maat_ip_plugin_table_get_ex_data(struct maat *instance, int table_id, int maat_ip_plugin_table_get_ex_data(struct maat *instance, int table_id,
const struct ip_addr *ip, void **ex_data_array, const struct ip_addr *ip, void **ex_data_array,
size_t n_ex_data); size_t n_ex_data);
@@ -155,29 +157,30 @@ struct maat_state;
* MAAT_SCAN_HIT * MAAT_SCAN_HIT
*/ */
int maat_scan_flag(struct maat *instance, int table_id, int thread_id, int maat_scan_flag(struct maat *instance, int table_id, int thread_id,
uint64_t flag, int *results, size_t n_result, uint64_t flag, uint64_t *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state); size_t *n_hit_result, struct maat_state **state);
int maat_scan_integer(struct maat *instance, int table_id, int thread_id, int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, int *results, size_t n_result, unsigned int intval, uint64_t *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state); size_t *n_hit_result, struct maat_state **state);
int maat_scan_ipv4(struct maat *instance, int table_id, int thread_id, int maat_scan_ipv4(struct maat *instance, int table_id, int thread_id,
uint32_t ip_addr, int *results, size_t n_result, uint32_t ip_addr, uint64_t *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state); size_t *n_hit_result, struct maat_state **state);
int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id, int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id,
uint8_t *ip_addr, int *results, size_t n_result, uint8_t *ip_addr, uint64_t *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state); size_t *n_hit_result, struct maat_state **state);
int maat_scan_string(struct maat *instance, int table_id, int thread_id, int maat_scan_string(struct maat *instance, int table_id, int thread_id,
const char *data, size_t data_len, int *results, size_t n_result, const char *data, size_t data_len, uint64_t *results,
size_t *n_hit_result, struct maat_state **state); size_t n_result, size_t *n_hit_result,
struct maat_state **state);
struct maat_stream; struct maat_stream;
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id); struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id);
int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* data, int data_len, int maat_scan_stream(struct maat_stream **stream, const char* data, int data_len,
int *results, size_t *n_result, struct maat_state **state); uint64_t *results, size_t *n_result, struct maat_state **state);
void maat_scan_stream_close(struct maat_stream **stream); void maat_scan_stream_close(struct maat_stream **stream);

7
scanner/adapter_hs.cpp
View File

@@ -424,8 +424,7 @@ int matched_event_cb(unsigned int id, unsigned long long from,
matched_pat->matched_l_offset = from; matched_pat->matched_l_offset = from;
matched_pat->matched_r_offset = to; matched_pat->matched_r_offset = to;
HASH_ADD_INT(matched_pat_container->pat_hash, pattern_id, matched_pat); HASH_ADD(hh, matched_pat_container->pat_hash, pattern_id, sizeof(unsigned long long), matched_pat);
return 0; return 0;
} }
@@ -471,8 +470,8 @@ int hs_tag_validate(struct hs_tag *hs_tag, struct matched_pattern_container *mat
/* check if real matched pattern, because pattern match_mode is different */ /* check if real matched pattern, because pattern match_mode is different */
for (size_t i = 0; i < hs_tag->n_pat_attr; i++) { for (size_t i = 0; i < hs_tag->n_pat_attr; i++) {
struct matched_pattern *tmp_matched_pat = NULL; struct matched_pattern *tmp_matched_pat = NULL;
int pattern_id = hs_tag->pat_attr[i].pattern_id; unsigned long long pattern_id = hs_tag->pat_attr[i].pattern_id;
HASH_FIND_INT(matched_pat_container->pat_hash, &pattern_id, tmp_matched_pat); HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), tmp_matched_pat);
if (tmp_matched_pat) { if (tmp_matched_pat) {
int matched_ret = is_real_matched_pattern(tmp_matched_pat, int matched_ret = is_real_matched_pattern(tmp_matched_pat,
hs_tag->pat_attr[i].match_mode, hs_tag->pat_attr[i].match_mode,

2
scanner/adapter_hs.h
View File

@@ -54,7 +54,7 @@ enum hs_case_sensitive {
}; };
struct hs_scan_result { struct hs_scan_result {
int item_id; uint64_t item_id;
void *user_tag; void *user_tag;
}; };

4
src/inc_internal/maat_command.h
View File

@@ -28,7 +28,7 @@ struct maat_cmd_line
{ {
const char *table_name; const char *table_name;
const char *table_line; const char *table_line;
int rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary. uint64_t rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary.
int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout. int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout.
}; };
@@ -41,6 +41,8 @@ struct maat_cmd_line
*/ */
int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule); int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule);
long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int increment);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

7
src/inc_internal/maat_compile.h
View File

@@ -53,7 +53,7 @@ int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema,
maat_rule_ex_dup_func_t *dup_func, maat_rule_ex_dup_func_t *dup_func,
long argl, void *argp, long argl, void *argp,
struct log_handle *logger); struct log_handle *logger);
void *compile_table_get_rule_ex_data(struct compile_schema *compile_schema, int compile_id, size_t idx); void *compile_table_get_rule_ex_data(struct compile_schema *compile_schema, uint64_t compile_id, size_t idx);
void compile_table_rule_ex_data_iterate(struct compile_schema *compile_schema, int idx); void compile_table_rule_ex_data_iterate(struct compile_schema *compile_schema, int idx);
size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema); size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema);
@@ -68,7 +68,7 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
const char *line, int valid_column); const char *line, int valid_column);
int compile_runtime_commit(void *compile_runtime, const char *table_name); int compile_runtime_commit(void *compile_runtime, const char *table_name);
int compile_runtime_match(struct compile_runtime *compile_rt, int *compile_ids, int compile_runtime_match(struct compile_runtime *compile_rt, uint64_t *compile_ids,
size_t compile_ids_size, struct maat_state *state); size_t compile_ids_size, struct maat_state *state);
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
@@ -92,8 +92,7 @@ struct maat_compile_state *maat_compile_state_new(int thread_id);
void maat_compile_state_free(struct maat_compile_state *compile_state); void maat_compile_state_free(struct maat_compile_state *compile_state);
int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
int *hit_item_ids, size_t hit_item_cnt, uint64_t *hit_item_ids, size_t hit_item_cnt,
int *group_ids, int hit_group_index, size_t group_ids_size,
size_t *n_hit_group_id, struct maat_state *state); size_t *n_hit_group_id, struct maat_state *state);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);

4
src/inc_internal/maat_expr.h
View File

@@ -45,12 +45,10 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name);
* @retval the num of hit group_id * @retval the num of hit group_id
*/ */
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int *group_ids, int hit_group_index, size_t data_len, int vtable_ids, struct maat_state *state);
size_t group_ids_size, int vtable_ids, struct maat_state *state);
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len,
int *group_ids, int hit_group_index, size_t group_ids_size,
int vtable_id, struct maat_state *state); int vtable_id, struct maat_state *state);
void expr_runtime_stream_close(struct expr_runtime *expr_rt); void expr_runtime_stream_close(struct expr_runtime *expr_rt);

1
src/inc_internal/maat_flag.h
View File

@@ -47,7 +47,6 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name);
* @retval the num of hit group_id * @retval the num of hit group_id
*/ */
int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, uint64_t flag, int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, uint64_t flag,
int *group_ids, int hit_group_index,size_t group_ids_size,
int vtable_id, struct maat_state *state); int vtable_id, struct maat_state *state);
void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id); void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id);

10
src/inc_internal/maat_group.h
View File

@@ -16,6 +16,8 @@ extern "C"
{ {
#endif #endif
#include <stdint.h>
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
#include "maat_kv.h" #include "maat_kv.h"
@@ -38,13 +40,13 @@ void group2group_runtime_free(void *g2g_runtime);
void maat_group_ref_inc(struct maat_group *group); void maat_group_ref_inc(struct maat_group *group);
void maat_group_ref_dec(struct maat_group *group); void maat_group_ref_dec(struct maat_group *group);
struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id); struct maat_group *group2group_runtime_add_group(void *g2g_runtime, uint64_t group_id);
void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group); void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group);
struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id); struct maat_group *group2group_runtime_find_group(void *g2g_runtime, uint64_t group_id);
int group2group_runtime_build_top_groups(void *g2g_runtime); int group2group_runtime_build_top_groups(void *g2g_runtime);
int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, int group2group_runtime_get_top_groups(void *g2g_runtime, uint64_t *group_ids,
size_t n_group_ids, int *top_group_ids); size_t n_group_ids, uint64_t *top_group_ids);
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
const char *line, int valid_column); const char *line, int valid_column);

3
src/inc_internal/maat_interval.h
View File

@@ -47,8 +47,7 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name);
* @retval the num of hit group_id * @retval the num of hit group_id
*/ */
int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
uint64_t integer, int *group_ids, int hit_group_index, uint64_t integer, int vtable_id, struct maat_state *state);
size_t group_ids_size, int vtable_id, struct maat_state *state);
void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id); void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id);
long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread); long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread);

3
src/inc_internal/maat_ip.h
View File

@@ -41,8 +41,7 @@ struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt);
/* ip runtime scan API */ /* ip runtime scan API */
int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
uint8_t *ip_addr, int *group_ids, int hit_group_index, uint8_t *ip_addr, int vtable_id, struct maat_state *state);
size_t group_id_size, int vtable_id, struct maat_state *state);
void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id); void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id);
long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread); long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread);

2
src/inc_internal/maat_plugin.h
View File

@@ -64,6 +64,8 @@ size_t plugin_runtime_cached_row_count(void *plugin_runtime);
const char *plugin_runtime_cached_row_get(void *plugin_runtime, size_t index); const char *plugin_runtime_cached_row_get(void *plugin_runtime, size_t index);
void *plugin_runtime_get_ex_data(void *plugin_runtime, void *plugin_schema, const char *key);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

20
src/inc_internal/maat_rule.h
View File

@@ -61,7 +61,7 @@ enum tag_match {
}; };
struct maat_rule_head { struct maat_rule_head {
int config_id; uint64_t config_id;
char resevered; char resevered;
int serv_def_len; int serv_def_len;
}; };
@@ -69,8 +69,8 @@ struct maat_rule_head {
#define ITEM_RULE_MAGIC 0x4d3c2b1a #define ITEM_RULE_MAGIC 0x4d3c2b1a
struct maat_item_inner { struct maat_item_inner {
long long magic_num; long long magic_num;
int item_id; uint64_t item_id;
int group_id; uint64_t group_id;
int district_id; int district_id;
int expr_id_cnt; int expr_id_cnt;
int expr_id_lb; //low boundary int expr_id_lb; //low boundary
@@ -78,8 +78,8 @@ struct maat_item_inner {
}; };
struct maat_item { struct maat_item {
int item_id; uint64_t item_id;
int group_id; uint64_t group_id;
UT_hash_handle hh; UT_hash_handle hh;
void *user_data; void *user_data;
}; };
@@ -87,7 +87,7 @@ struct maat_item {
#define COMPILE_RULE_MAGIC 0x1a2b3c4d #define COMPILE_RULE_MAGIC 0x1a2b3c4d
struct compile_rule { struct compile_rule {
long long magic_num; long long magic_num;
int compile_id; uint64_t compile_id;
struct maat_rule_head head;// fix len of Maat_rule_t struct maat_rule_head head;// fix len of Maat_rule_t
char *service_defined; char *service_defined;
int declared_clause_num; int declared_clause_num;
@@ -98,8 +98,8 @@ struct compile_rule {
}; };
struct group2group_rule { struct group2group_rule {
int group_id; uint64_t group_id;
int super_group_id; uint64_t super_group_id;
}; };
struct maat_runtime { struct maat_runtime {
@@ -254,11 +254,11 @@ size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_
int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag); int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag);
struct maat_item *maat_item_new(int item_id, int group_id, void *user_data); struct maat_item *maat_item_new(uint64_t item_id, uint64_t group_id, void *user_data);
void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *)); void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *));
struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id); struct maat_item_inner *maat_item_inner_new(uint64_t group_id, uint64_t item_id, int district_id);
void maat_item_inner_free(void *item_inner); void maat_item_inner_free(void *item_inner);

2
src/inc_internal/maat_utils.h
View File

@@ -80,7 +80,7 @@ void ipv6_ntoh(unsigned int *v6_addr);
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len); int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len);
/* the column value must be integer */ /* the column value must be integer */
int get_column_value(const char *line, int column_seq); long long get_column_value(const char *line, int column_seq);
int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *out_sz); int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *out_sz);

789
src/maat_api.c
View File

File diff suppressed because it is too large Load Diff

14
src/maat_bool_plugin.c
View File

@@ -33,7 +33,7 @@ struct bool_plugin_schema {
}; };
struct bool_plugin_item { struct bool_plugin_item {
int item_id; uint64_t item_id;
size_t n_bool_item; size_t n_bool_item;
unsigned long long bool_item_id[MAX_ITEMS_PER_BOOL_EXPR]; unsigned long long bool_item_id[MAX_ITEMS_PER_BOOL_EXPR];
}; };
@@ -290,7 +290,7 @@ bool_plugin_item_new(const char *line, struct bool_plugin_schema *schema,
schema->table_id, line); schema->table_id, line);
goto error; goto error;
} }
item->item_id = atoi(line + column_offset); item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, schema->bool_expr_column, &column_offset, &column_len); ret = get_column_pos(line, schema->bool_expr_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -355,7 +355,7 @@ size_t ull_dedup(unsigned long long item_ids[], size_t n_item)
return index + 1; return index + 1;
} }
struct bool_expr *bool_expr_new(int item_id, struct bool_plugin_item *item) struct bool_expr *bool_expr_new(uint64_t item_id, struct bool_plugin_item *item)
{ {
struct bool_expr *expr = ALLOC(struct bool_expr, 1); struct bool_expr *expr = ALLOC(struct bool_expr, 1);
@@ -383,7 +383,11 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche
struct bool_expr *expr = NULL; struct bool_expr *expr = NULL;
struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema; struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema;
struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime;
int item_id = get_column_value(line, schema->item_id_column); long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
return -1;
}
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
@@ -404,7 +408,7 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche
char *key = (char *)&item_id; char *key = (char *)&item_id;
int ret = bool_plugin_runtime_update_row(bool_plugin_rt, schema, line, key, int ret = bool_plugin_runtime_update_row(bool_plugin_rt, schema, line, key,
sizeof(int), expr, is_valid); sizeof(uint64_t), expr, is_valid);
if (ret < 0) { if (ret < 0) {
if (item != NULL) { if (item != NULL) {
FREE(item); FREE(item);

22
src/maat_command.c
View File

@@ -353,3 +353,25 @@ error_out:
return ret; return ret;
} }
long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int increment)
{
long long result = 0;
redisContext *write_ctx = get_redis_ctx_for_write(maat_instance);
if (NULL == write_ctx) {
return -1;
}
redisReply *data_reply = maat_cmd_wrap_redis_command(write_ctx, "INCRBY %s %d", key, increment);
if (data_reply->type == REDIS_REPLY_INTEGER) {
result = data_reply->integer;
} else {
result = -1;
}
freeReplyObject(data_reply);
data_reply = NULL;
return result;
}

165
src/maat_compile.c
View File

@@ -29,7 +29,7 @@
#define MAX_COMPILE_EX_DATA_NUM 2 #define MAX_COMPILE_EX_DATA_NUM 2
enum user_region_encode { enum user_region_encode {
USER_REGION_ENCODE_NONE=0, USER_REGION_ENCODE_NONE = 0,
USER_REGION_ENCODE_ESCAPE, USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64 USER_REGION_ENCODE_BASE64
}; };
@@ -60,15 +60,15 @@ struct group2compile_schema {
}; };
struct compile_item { struct compile_item {
int compile_id; uint64_t compile_id;
char user_region[MAX_TABLE_LINE_SIZE]; char user_region[MAX_TABLE_LINE_SIZE];
int declared_clause_num; int declared_clause_num;
int evaluation_order; int evaluation_order;
}; };
struct group2compile_item { struct group2compile_item {
int group_id; uint64_t group_id;
int compile_id; uint64_t compile_id;
int not_flag; int not_flag;
int vtable_id; int vtable_id;
int clause_index; int clause_index;
@@ -104,7 +104,7 @@ struct maat_clause_state {
}; };
struct maat_literal_id { struct maat_literal_id {
int group_id; uint64_t group_id;
int vtable_id; int vtable_id;
}; };
@@ -118,14 +118,14 @@ struct maat_clause {
struct compile_sort_para { struct compile_sort_para {
double evaluation_order; double evaluation_order;
int declared_clause_num; int declared_clause_num;
int compile_id; uint64_t compile_id;
void *user; void *user;
}; };
#define MAAT_COMPILE_MAGIC 0x4a5b6c7d #define MAAT_COMPILE_MAGIC 0x4a5b6c7d
struct maat_compile { struct maat_compile {
unsigned int magic; unsigned int magic;
int compile_id; uint64_t compile_id;
int actual_clause_num; int actual_clause_num;
int declared_clause_num; int declared_clause_num;
int not_clause_cnt; int not_clause_cnt;
@@ -138,8 +138,8 @@ struct maat_compile {
struct maat_internal_hit_path { struct maat_internal_hit_path {
int Nth_scan; int Nth_scan;
int Nth_hit_item; int Nth_hit_item;
int item_id; uint64_t item_id;
int group_id; uint64_t group_id;
int vtable_id; int vtable_id;
}; };
@@ -184,13 +184,13 @@ int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema,
return idx; return idx;
} }
void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, int compile_id, int is_dettach) void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, uint64_t compile_id, int is_dettach)
{ {
struct maat_compile *compile = NULL; struct maat_compile *compile = NULL;
void *ret = NULL; void *ret = NULL;
pthread_rwlock_rdlock(&compile_rt->rwlock); pthread_rwlock_rdlock(&compile_rt->rwlock);
HASH_FIND_INT(compile_rt->compile_hash, &compile_id, compile); HASH_FIND(hh, compile_rt->compile_hash, &compile_id, sizeof(uint64_t), compile);
if (compile != NULL) { if (compile != NULL) {
ret = compile->user_data; ret = compile->user_data;
if (is_dettach) { if (is_dettach) {
@@ -278,7 +278,7 @@ void compile_table_rule_ex_data_iterate(struct compile_schema *compile_schema, i
compile_runtime_user_data_iterate(compile_rt, rule_ex_data_new_cb, ex_schema); compile_runtime_user_data_iterate(compile_rt, rule_ex_data_new_cb, ex_schema);
} }
void *compile_table_get_rule_ex_data(struct compile_schema *compile_schema, int compile_id, size_t idx) void *compile_table_get_rule_ex_data(struct compile_schema *compile_schema, uint64_t compile_id, size_t idx)
{ {
if (NULL == compile_schema) { if (NULL == compile_schema) {
return NULL; return NULL;
@@ -495,7 +495,7 @@ compile_item_new(const char *line, struct compile_schema *compile_schema,
compile_schema->table_id, line); compile_schema->table_id, line);
goto error; goto error;
} }
compile_item->compile_id = atoi(line + column_offset); compile_item->compile_id = atoll(line + column_offset);
ret = get_column_pos(line, compile_schema->tags_column, ret = get_column_pos(line, compile_schema->tags_column,
&column_offset, &column_len); &column_offset, &column_len);
@@ -717,7 +717,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
g2c_schema->table_id, line); g2c_schema->table_id, line);
goto error; goto error;
} }
g2c_item->group_id = atoi(line + column_offset); g2c_item->group_id = atoll(line + column_offset);
ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset, &column_len); ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -726,7 +726,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
g2c_schema->table_id, line); g2c_schema->table_id, line);
goto error; goto error;
} }
g2c_item->compile_id = atoi(line + column_offset); g2c_item->compile_id = atoll(line + column_offset);
ret = get_column_pos(line, g2c_schema->not_flag_column, &column_offset, &column_len); ret = get_column_pos(line, g2c_schema->not_flag_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -790,7 +790,7 @@ void group2compile_item_free(struct group2compile_item *g2c_item)
} }
#define MAAT_HIER_COMPILE_MAGIC 0x4a5b6c7d #define MAAT_HIER_COMPILE_MAGIC 0x4a5b6c7d
struct maat_compile *maat_compile_new(int compile_id) struct maat_compile *maat_compile_new(uint64_t compile_id)
{ {
struct maat_compile *compile = ALLOC(struct maat_compile, 1); struct maat_compile *compile = ALLOC(struct maat_compile, 1);
@@ -819,31 +819,31 @@ int maat_compile_set(struct maat_compile *compile, int declared_clause_num,
return 0; return 0;
} }
int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id, int maat_compile_hash_add(struct maat_compile **compile_hash, uint64_t compile_id,
struct maat_compile *compile) struct maat_compile *compile)
{ {
int ret = 0; int ret = 0;
assert(compile->declared_clause_num >= 0); assert(compile->declared_clause_num >= 0);
HASH_ADD_INT(*compile_hash, compile_id, compile); HASH_ADD(hh, *compile_hash, compile_id, sizeof(uint64_t), compile);
//TODO:mytest need to delete //TODO:mytest need to delete
#if 0 #if 0
size_t compile_cnt = HASH_COUNT(*compile_hash); size_t compile_cnt = HASH_COUNT(*compile_hash);
struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL; struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL;
HASH_ITER (hh, *compile_hash, compile1, tmp_compile1) { HASH_ITER (hh, *compile_hash, compile1, tmp_compile1) {
printf("compile->compile_id:%d, compile_cnt:%zu\n", compile1->compile_id, compile_cnt); printf("<maat_compile_hash_add> compile_id:%lu, compile_cnt:%zu\n",
compile1->compile_id, compile_cnt);
} }
#endif #endif
return ret; return ret;
} }
void maat_compile_hash_set(struct maat_compile **compile_hash, int compile_id, void maat_compile_hash_set(struct maat_compile **compile_hash, uint64_t compile_id,
struct maat_compile *compile) struct maat_compile *compile)
{ {
struct maat_compile *tmp_compile = NULL; struct maat_compile *tmp_compile = NULL;
HASH_FIND_INT(*compile_hash, &compile_id, tmp_compile); HASH_FIND(hh, *compile_hash, &compile_id, sizeof(uint64_t), tmp_compile);
assert(tmp_compile != NULL); assert(tmp_compile != NULL);
assert(tmp_compile->user_data == NULL); assert(tmp_compile->user_data == NULL);
@@ -852,8 +852,8 @@ void maat_compile_hash_set(struct maat_compile **compile_hash, int compile_id,
} }
int maat_compile_hash_remove(struct maat_compile **compile_hash, int compile_id, int maat_compile_hash_remove(struct maat_compile **compile_hash, struct maat_compile *compile,
struct maat_compile *compile, struct maat_garbage_bin *garbage_bin) struct maat_garbage_bin *garbage_bin)
{ {
if (0 == compile->actual_clause_num) { if (0 == compile->actual_clause_num) {
HASH_DEL(*compile_hash, compile); HASH_DEL(*compile_hash, compile);
@@ -865,20 +865,20 @@ int maat_compile_hash_remove(struct maat_compile **compile_hash, int compile_id,
size_t compile_cnt = HASH_COUNT(*compile_hash); size_t compile_cnt = HASH_COUNT(*compile_hash);
struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL; struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL;
HASH_ITER (hh, *compile_hash, compile1, tmp_compile1) { HASH_ITER (hh, *compile_hash, compile1, tmp_compile1) {
printf("compile->compile_id:%d, compile_cnt:%zu\n", compile1->compile_id, compile_cnt); printf("<maat_compile_hash_remove> compile_id:%lu, compile_cnt:%zu\n",
compile1->compile_id, compile_cnt);
} }
#endif #endif
return 0; return 0;
} }
struct maat_compile *maat_compile_hash_find(struct maat_compile **compile_hash, struct maat_compile *maat_compile_hash_find(struct maat_compile **compile_hash, uint64_t compile_id)
int compile_id)
{ {
struct maat_compile *compile = NULL; struct maat_compile *compile = NULL;
HASH_FIND(hh, *compile_hash, &compile_id, sizeof(compile_id), compile); HASH_FIND(hh, *compile_hash, &compile_id, sizeof(compile_id), compile);
return compile; return compile;
} }
size_t maat_compile_in_use_count(struct maat_compile *compile_hash) size_t maat_compile_in_use_count(struct maat_compile *compile_hash)
@@ -930,7 +930,8 @@ int maat_compile_clause_add_literal(struct maat_compile *compile,
tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids,
literal_id, compare_literal_id); literal_id, compare_literal_id);
if (tmp) { if (tmp) {
assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); assert(tmp->group_id == literal_id->group_id);
assert(tmp->vtable_id == literal_id->vtable_id);
return -1; return -1;
} else { } else {
utarray_push_back(clause_state->literal_ids, literal_id); utarray_push_back(clause_state->literal_ids, literal_id);
@@ -1055,7 +1056,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
#if 0 #if 0
struct maat_literal_id *p = NULL; struct maat_literal_id *p = NULL;
for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) { for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) {
printf("compile_id:%d, clause_id:%llu, literal{%d: %d}\n", printf("compile_id:%lu, clause_id:%llu, literal{%lu: %d}\n",
compile->compile_id, compile->clause_states[i].clause_id, p->group_id, p->vtable_id); compile->compile_id, compile->clause_states[i].clause_id, p->group_id, p->vtable_id);
} }
#endif #endif
@@ -1066,6 +1067,8 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
} }
} }
// printf("bool_matcher_new compile_id:%lu j:%zu, compile->declared_clause_num:%d\n",
// compile->compile_id, j, compile->declared_clause_num);
//some compile may have zero groups, e.g. default policy. //some compile may have zero groups, e.g. default policy.
if (j == (size_t)compile->declared_clause_num && j > 0) { if (j == (size_t)compile->declared_clause_num && j > 0) {
bool_expr_array[expr_cnt].expr_id = compile->compile_id; bool_expr_array[expr_cnt].expr_id = compile->compile_id;
@@ -1163,14 +1166,9 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan
return ud_result_cnt; return ud_result_cnt;
} }
int maat_add_group_to_compile(struct maat_compile **compile_hash, void *g2g_runtime, int maat_add_group_to_compile(struct maat_compile **compile_hash, struct group2compile_item *g2c_item,
struct group2compile_item *g2c_item, struct log_handle *logger) struct log_handle *logger)
{ {
struct maat_group *group = group2group_runtime_find_group(g2g_runtime, g2c_item->group_id);
if (!group) {
group = group2group_runtime_add_group(g2g_runtime, g2c_item->group_id);
}
int ret = -1; int ret = -1;
struct maat_compile *compile = maat_compile_hash_find(compile_hash, g2c_item->compile_id); struct maat_compile *compile = maat_compile_hash_find(compile_hash, g2c_item->compile_id);
if (!compile) { if (!compile) {
@@ -1192,27 +1190,19 @@ int maat_add_group_to_compile(struct maat_compile **compile_hash, void *g2g_runt
g2c_item->compile_id); g2c_item->compile_id);
ret = -1; ret = -1;
} else { } else {
maat_group_ref_inc(group);
ret = 0; ret = 0;
} }
// printf("group2compile update compile_id:%lu, compile->declared_clause_num:%d\n",
// compile->compile_id, compile->declared_clause_num);
return ret; return ret;
} }
int maat_remove_group_from_compile(struct maat_compile **compile_hash, int maat_remove_group_from_compile(struct maat_compile **compile_hash,
void *g2g_runtime,
struct group2compile_item *g2c_item, struct group2compile_item *g2c_item,
struct maat_garbage_bin *garbage_bin, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
struct maat_group *group = group2group_runtime_find_group(g2g_runtime, g2c_item->group_id);
if (!group) {
log_error(logger, MODULE_COMPILE,
"Remove group %d from compile %d failed, group is not exisited.",
g2c_item->group_id, g2c_item->compile_id);
return -1;
}
struct maat_compile *compile = NULL; struct maat_compile *compile = NULL;
HASH_FIND(hh, *compile_hash, &(g2c_item->compile_id), sizeof(g2c_item->compile_id), compile); HASH_FIND(hh, *compile_hash, &(g2c_item->compile_id), sizeof(g2c_item->compile_id), compile);
if (!compile) { if (!compile) {
@@ -1230,7 +1220,6 @@ int maat_remove_group_from_compile(struct maat_compile **compile_hash,
g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, g2c_item->compile_id); g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, g2c_item->compile_id);
return -1; return -1;
} }
maat_group_ref_dec(group);
if (0 == compile->actual_clause_num && NULL == compile->user_data) { if (0 == compile->actual_clause_num && NULL == compile->user_data) {
HASH_DEL(*compile_hash, compile); HASH_DEL(*compile_hash, compile);
@@ -1274,7 +1263,7 @@ void maat_compile_state_free(struct maat_compile_state *compile_state)
free(compile_state); free(compile_state);
} }
static int maat_compile_hit_path_add(UT_array *hit_paths, int item_id, int group_id, static int maat_compile_hit_path_add(UT_array *hit_paths, uint64_t item_id, uint64_t group_id,
int vtable_id, int Nth_scan, int Nth_item_result) int vtable_id, int Nth_scan, int Nth_item_result)
{ {
struct maat_internal_hit_path new_path; struct maat_internal_hit_path new_path;
@@ -1343,7 +1332,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
/* /*
NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths
*/ */
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; uint64_t top_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
memset(top_group_ids, 0, sizeof(top_group_ids));
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &(internal_path->group_id), int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &(internal_path->group_id),
1, top_group_ids); 1, top_group_ids);
if (top_group_cnt <= 0) { if (top_group_cnt <= 0) {
@@ -1417,7 +1407,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
} }
void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state,
int item_id, int group_id, int vtable_id, uint64_t item_id, uint64_t group_id, int vtable_id,
int Nth_scan, int Nth_item_result) int Nth_scan, int Nth_item_result)
{ {
if (compile_state->Nth_scan != Nth_scan) { if (compile_state->Nth_scan != Nth_scan) {
@@ -1434,7 +1424,7 @@ void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state
} }
void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state,
void *compile_runtime, int group_id, void *compile_runtime, uint64_t group_id,
int vtable_id) int vtable_id)
{ {
if (NULL == compile_state || NULL == compile_runtime) { if (NULL == compile_state || NULL == compile_runtime) {
@@ -1448,7 +1438,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
unsigned long long *clause_id = 0; unsigned long long *clause_id = 0;
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
pthread_rwlock_wrlock(&compile_rt->rwlock); pthread_rwlock_rdlock(&compile_rt->rwlock);
assert(compile_rt->compile_hash != NULL); assert(compile_rt->compile_hash != NULL);
HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) { HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) {
@@ -1554,11 +1544,15 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
struct compile_schema *schema = (struct compile_schema *)compile_schema; struct compile_schema *schema = (struct compile_schema *)compile_schema;
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
int compile_id = get_column_value(line, schema->compile_id_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
} }
long long compile_id = get_column_value(line, schema->compile_id_column);
if (compile_id < 0) {
return -1;
}
if (0 == is_valid) { if (0 == is_valid) {
//delete //delete
pthread_rwlock_wrlock(&compile_rt->rwlock); pthread_rwlock_wrlock(&compile_rt->rwlock);
@@ -1568,12 +1562,12 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
return -1; return -1;
} }
ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile_id, ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile,
compile, compile_rt->ref_garbage_bin); compile_rt->ref_garbage_bin);
pthread_rwlock_unlock(&compile_rt->rwlock); pthread_rwlock_unlock(&compile_rt->rwlock);
if (ret < 0) { if (ret < 0) {
log_error(compile_rt->logger, MODULE_COMPILE, log_error(compile_rt->logger, MODULE_COMPILE,
"remove compile table(table_id:%d) compile(compile_id:%d) from compile_hash failed", "remove compile table(table_id:%d) compile(compile_id:%lld) from compile_hash failed",
schema->table_id, compile_id); schema->table_id, compile_id);
return -1; return -1;
} }
@@ -1604,16 +1598,15 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule, maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule,
(void (*)(void *))destroy_compile_rule); (void (*)(void *))destroy_compile_rule);
struct maat_compile *tmp_compile = maat_compile_hash_find(&(compile_rt->compile_hash), compile_id); struct maat_compile *tmp_compile = maat_compile_hash_find(&(compile_rt->compile_hash), compile_id);
if (tmp_compile != NULL) { if (tmp_compile != NULL) {
maat_compile_hash_set(&(compile_rt->compile_hash), compile_id, compile); maat_compile_hash_set(&(compile_rt->compile_hash), compile_id, compile);
} else { } else {
maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile); maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile);
} }
if (compile->compile_id == 141) { // printf("compile_runtime_update compile_id:%lu, compile->declared_clause_num:%d\n",
printf("compile->declared_clause_num:%d\n", compile->declared_clause_num); // compile->compile_id, compile->declared_clause_num);
}
pthread_rwlock_unlock(&compile_rt->rwlock); pthread_rwlock_unlock(&compile_rt->rwlock);
} }
@@ -1642,27 +1635,45 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
return -1; return -1;
} }
pthread_rwlock_wrlock(&compile_rt->rwlock); struct maat_group *group = NULL;
if (0 == is_valid) { if (0 == is_valid) {
//delete //delete
ret = maat_remove_group_from_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item, group = group2group_runtime_find_group(g2g_rt, g2c_item->group_id);
if (!group) {
log_error(compile_rt->logger, MODULE_COMPILE,
"Remove group %d from compile %d failed, group is not exisited.",
g2c_item->group_id, g2c_item->compile_id);
return -1;
}
pthread_rwlock_wrlock(&compile_rt->rwlock);
ret = maat_remove_group_from_compile(&(compile_rt->compile_hash), g2c_item,
compile_rt->ref_garbage_bin, compile_rt->logger); compile_rt->ref_garbage_bin, compile_rt->logger);
pthread_rwlock_unlock(&compile_rt->rwlock);
if (0 == ret) { if (0 == ret) {
if (g2c_item->not_flag) { if (g2c_item->not_flag) {
g2c_rt->not_flag_group--; g2c_rt->not_flag_group--;
} }
maat_group_ref_dec(group);
} }
} else { } else {
//add //add
ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item, group = group2group_runtime_find_group(g2g_rt, g2c_item->group_id);
compile_rt->logger); if (!group) {
group = group2group_runtime_add_group(g2g_rt, g2c_item->group_id);
}
pthread_rwlock_wrlock(&compile_rt->rwlock);
ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2c_item, compile_rt->logger);
pthread_rwlock_unlock(&compile_rt->rwlock);
if (0 == ret) { if (0 == ret) {
if (g2c_item->not_flag) { if (g2c_item->not_flag) {
g2c_rt->not_flag_group++; g2c_rt->not_flag_group++;
} }
maat_group_ref_inc(group);
} }
} }
pthread_rwlock_unlock(&compile_rt->rwlock);
group2compile_item_free(g2c_item); group2compile_item_free(g2c_item);
return ret; return ret;
@@ -1758,7 +1769,7 @@ static int compare_compile_rule(const void *a, const void *b)
} }
int compile_runtime_match(struct compile_runtime *compile_rt, int compile_runtime_match(struct compile_runtime *compile_rt,
int *compile_ids, size_t compile_ids_size, uint64_t *compile_ids, size_t compile_ids_size,
struct maat_state *state) struct maat_state *state)
{ {
struct maat_compile_state *compile_state = state->compile_state; struct maat_compile_state *compile_state = state->compile_state;
@@ -1785,12 +1796,13 @@ int compile_runtime_match(struct compile_runtime *compile_rt,
} }
int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
int *hit_item_ids, size_t hit_item_cnt, uint64_t *hit_item_ids, size_t hit_item_cnt,
int *group_ids, int hit_group_index, size_t group_ids_size,
size_t *n_hit_group_id, struct maat_state *state) size_t *n_hit_group_id, struct maat_state *state)
{ {
struct maat_item *item = NULL; struct maat_item *item = NULL;
size_t hit_group_cnt = hit_group_index; uint64_t hit_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
memset(hit_group_ids, 0, sizeof(hit_group_ids));
size_t hit_group_cnt = 0;
void *g2g_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, void *g2g_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
state->maat_instance->g2g_table_id); state->maat_instance->g2g_table_id);
@@ -1799,18 +1811,18 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
} }
for (size_t i = 0; i < hit_item_cnt; i++) { for (size_t i = 0; i < hit_item_cnt; i++) {
HASH_FIND_INT(item_hash, &(hit_item_ids[i]), item); HASH_FIND(hh, item_hash, &(hit_item_ids[i]), sizeof(uint64_t), item);
//assert(item != NULL); //assert(item != NULL);
if (!item) { if (!item) {
// item config has been deleted // item config has been deleted
continue; continue;
} }
if (hit_group_cnt >= group_ids_size) { if (hit_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
hit_group_cnt = group_ids_size; hit_group_cnt = MAX_SCANNER_HIT_GROUP_NUM;
//Prevent group_id_array out of bounds //Prevent group_id_array out of bounds
} else { } else {
group_ids[hit_group_cnt++] = item->group_id; hit_group_ids[hit_group_cnt++] = item->group_id;
} }
// update hit path // update hit path
@@ -1837,8 +1849,9 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
compile_table_ids[idx]); compile_table_ids[idx]);
for (size_t i = 0; i < hit_group_cnt; i++) { for (size_t i = 0; i < hit_group_cnt; i++) {
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; uint64_t top_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i], memset(top_group_ids, 0, sizeof(top_group_ids));
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &hit_group_ids[i],
1, top_group_ids); 1, top_group_ids);
for (int j = 0; j < top_group_cnt; j++) { for (int j = 0; j < top_group_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt, maat_compile_state_update_hit_clause(state->compile_state, compile_rt,

50
src/maat_expr.c
View File

@@ -58,8 +58,8 @@ enum match_method {
}; };
struct expr_item { struct expr_item {
int item_id; uint64_t item_id;
int group_id; uint64_t group_id;
int district_id; int district_id;
char keywords[MAX_KEYWORDS_STR]; char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type; enum expr_type expr_type;
@@ -156,7 +156,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
expr_schema->table_id, line); expr_schema->table_id, line);
goto error; goto error;
} }
expr_item->item_id = atoi(line + column_offset); expr_item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -165,7 +165,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
expr_schema->table_id, line); expr_schema->table_id, line);
goto error; goto error;
} }
expr_item->group_id = atoi(line + column_offset); expr_item->group_id = atoll(line + column_offset);
table_type = table_manager_get_table_type(expr_schema->ref_tbl_mgr, expr_schema->table_id); table_type = table_manager_get_table_type(expr_schema->ref_tbl_mgr, expr_schema->table_id);
if (table_type == TABLE_TYPE_EXPR_PLUS) { if (table_type == TABLE_TYPE_EXPR_PLUS) {
@@ -485,7 +485,7 @@ void expr_runtime_free(void *expr_runtime)
} }
int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
size_t key_len, int item_id, and_expr_t *expr_rule, size_t key_len, uint64_t item_id, and_expr_t *expr_rule,
int is_valid, struct log_handle *logger) int is_valid, struct log_handle *logger)
{ {
void *data = NULL; void *data = NULL;
@@ -495,7 +495,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
data = rcu_hash_find(expr_rt->htable, key, key_len); data = rcu_hash_find(expr_rt->htable, key, key_len);
if (NULL == data) { if (NULL == data) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"the key of expr rule not exist, can't be deleted, expr_id:%d", "the key of expr rule not exist, can't be deleted, expr_id:%llu",
item_id); item_id);
return -1; return -1;
} }
@@ -505,7 +505,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
data = rcu_hash_find(expr_rt->htable, key, key_len); data = rcu_hash_find(expr_rt->htable, key, key_len);
if (data != NULL) { if (data != NULL) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"the key of expr rule already exist, can't be added, expr_id:%d", "the key of expr rule already exist, can't be added, expr_id:%llu",
item_id); item_id);
return -1; return -1;
} }
@@ -719,13 +719,17 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
struct expr_schema *schema = (struct expr_schema *)expr_schema; struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
int item_id = get_column_value(line, schema->item_id_column); long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
return -1;
}
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
} else if (0 == is_valid) { } else if (0 == is_valid) {
//delete //delete
HASH_FIND_INT(expr_rt->item_hash, &item_id, item); HASH_FIND(hh, expr_rt->item_hash, &item_id, sizeof(uint64_t), item);
if (NULL == item) { if (NULL == item) {
return -1; return -1;
} }
@@ -741,10 +745,10 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, maat_item_inner_free); maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else { } else {
//add //add
HASH_FIND_INT(expr_rt->item_hash, &item_id, item); HASH_FIND(hh, expr_rt->item_hash, &item_id, sizeof(uint64_t), item);
if (item) { if (item) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
"expr runtime add item %d to item_hash failed, already exist", "expr runtime add item %llu to item_hash failed, already exist",
item_id); item_id);
return -1; return -1;
} }
@@ -756,20 +760,20 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
u_para = maat_item_inner_new(expr_item->group_id, item_id, expr_item->district_id); u_para = maat_item_inner_new(expr_item->group_id, item_id, expr_item->district_id);
item = maat_item_new(item_id, expr_item->group_id, u_para); item = maat_item_new(item_id, expr_item->group_id, u_para);
HASH_ADD_INT(expr_rt->item_hash, item_id, item); HASH_ADD(hh, expr_rt->item_hash, item_id, sizeof(uint64_t), item);
expr_rule = expr_item_to_expr_rule(expr_item, u_para, expr_rt->logger); expr_rule = expr_item_to_expr_rule(expr_item, u_para, expr_rt->logger);
expr_item_free(expr_item); expr_item_free(expr_item);
if (NULL == expr_rule) { if (NULL == expr_rule) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
"transform expr table(table_id:%d) item to expr_rule failed, item_id:%d", "transform expr table(table_id:%d) item to expr_rule failed, item_id:%llu",
schema->table_id, item_id); schema->table_id, item_id);
return -1; return -1;
} }
} }
char *key = (char *)&item_id; char *key = (char *)&item_id;
ret = expr_runtime_update_row(expr_rt, key, sizeof(int), item_id, ret = expr_runtime_update_row(expr_rt, key, sizeof(uint64_t), item_id,
expr_rule, is_valid, expr_rt->logger); expr_rule, is_valid, expr_rt->logger);
if (ret < 0) { if (ret < 0) {
if (expr_rule != NULL) { if (expr_rule != NULL) {
@@ -841,8 +845,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
} }
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int *group_ids, int hit_group_index, size_t data_len, int vtable_ids, struct maat_state *state)
size_t group_ids_size, int vtable_ids, struct maat_state *state)
{ {
size_t n_hit_item = 0; size_t n_hit_item = 0;
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
@@ -865,12 +868,12 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
} }
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; uint64_t hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL; struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0; int real_hit_item_cnt = 0;
int district_id = state->district_id; int district_id = state->district_id;
memset(hit_item_ids, -1, sizeof(hit_item_ids)); memset(hit_item_ids, 0, sizeof(hit_item_ids));
for (size_t i = 0; i < n_hit_item; i++) { for (size_t i = 0; i < n_hit_item; i++) {
item = (struct maat_item_inner *)(hit_results[i].user_tag); item = (struct maat_item_inner *)(hit_results[i].user_tag);
@@ -881,8 +884,7 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
size_t group_hit_cnt = 0; size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_ids, hit_item_ids, ret = maat_compile_state_update(expr_rt->item_hash, vtable_ids, hit_item_ids,
real_hit_item_cnt, group_ids, hit_group_index, real_hit_item_cnt, &group_hit_cnt, state);
group_ids_size, &group_hit_cnt, state);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -901,7 +903,6 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
} }
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len,
int *group_ids, int hit_group_index, size_t group_ids_size,
int vtable_id, struct maat_state *state) int vtable_id, struct maat_state *state)
{ {
if (NULL == expr_rt) { if (NULL == expr_rt) {
@@ -928,15 +929,16 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, siz
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
} }
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; uint64_t hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
memset(hit_item_ids, 0, sizeof(hit_item_ids));
for (size_t i = 0; i < n_hit_item; i++) { for (size_t i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].item_id; hit_item_ids[i] = hit_results[i].item_id;
} }
size_t group_hit_cnt = 0; size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, hit_group_index, n_hit_item, &group_hit_cnt, state);
group_ids_size, &group_hit_cnt, state);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }

38
src/maat_flag.c
View File

@@ -34,8 +34,8 @@ struct flag_schema {
}; };
struct flag_item { struct flag_item {
int item_id; uint64_t item_id;
int group_id; uint64_t group_id;
uint64_t flag; uint64_t flag;
uint64_t flag_mask; uint64_t flag_mask;
}; };
@@ -176,7 +176,7 @@ void flag_runtime_free(void *flag_runtime)
} }
int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len, int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len,
int item_id, struct flag_rule *rule, int is_valid) uint64_t item_id, struct flag_rule *rule, int is_valid)
{ {
void *data = NULL; void *data = NULL;
@@ -185,7 +185,7 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_
data = rcu_hash_find(flag_rt->htable, key, key_len); data = rcu_hash_find(flag_rt->htable, key, key_len);
if (NULL == data) { if (NULL == data) {
log_error(flag_rt->logger, MODULE_FLAG, log_error(flag_rt->logger, MODULE_FLAG,
"the key of flag rule not exist, can't be deleted, item_id:%d", "the key of flag rule not exist, can't be deleted, item_id:%llu",
item_id); item_id);
return -1; return -1;
} }
@@ -195,7 +195,7 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_
data = rcu_hash_find(flag_rt->htable, key, key_len); data = rcu_hash_find(flag_rt->htable, key, key_len);
if (data != NULL) { if (data != NULL) {
log_error(flag_rt->logger, MODULE_FLAG, log_error(flag_rt->logger, MODULE_FLAG,
"the key of flag rule already exist, can't be added, item_id:%d", "the key of flag rule already exist, can't be added, item_id:%llu",
item_id); item_id);
return -1; return -1;
} }
@@ -219,7 +219,7 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *flag_schem
flag_schema->table_id, line); flag_schema->table_id, line);
goto error; goto error;
} }
flag_item->item_id = atoi(line + column_offset); flag_item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, flag_schema->group_id_column, &column_offset, &column_len); ret = get_column_pos(line, flag_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -228,7 +228,7 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *flag_schem
flag_schema->table_id, line); flag_schema->table_id, line);
goto error; goto error;
} }
flag_item->group_id = atoi(line + column_offset); flag_item->group_id = atoll(line + column_offset);
ret = get_column_pos(line, flag_schema->flag_column, &column_offset, &column_len); ret = get_column_pos(line, flag_schema->flag_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -291,13 +291,17 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
struct flag_schema *schema = (struct flag_schema *)flag_schema; struct flag_schema *schema = (struct flag_schema *)flag_schema;
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
int item_id = get_column_value(line, schema->item_id_column); long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
return -1;
}
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
} else if (0 == is_valid) { } else if (0 == is_valid) {
//delete //delete
HASH_FIND_INT(flag_rt->item_hash, &item_id, item); HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(uint64_t), item);
if (NULL == item) { if (NULL == item) {
return -1; return -1;
} }
@@ -313,7 +317,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free); maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else { } else {
//add //add
HASH_FIND_INT(flag_rt->item_hash, &item_id, item); HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(uint64_t), item);
if (item) { if (item) {
log_error(flag_rt->logger, MODULE_FLAG, log_error(flag_rt->logger, MODULE_FLAG,
"flag runtime add item %d to item_hash failed, already exist", "flag runtime add item %d to item_hash failed, already exist",
@@ -328,7 +332,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
u_para = maat_item_inner_new(flag_item->group_id, item_id, 0); u_para = maat_item_inner_new(flag_item->group_id, item_id, 0);
item = maat_item_new(item_id, flag_item->group_id, u_para); item = maat_item_new(item_id, flag_item->group_id, u_para);
HASH_ADD_INT(flag_rt->item_hash, item_id, item); HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(uint64_t), item);
flag_rule = flag_item_to_flag_rule(flag_item, u_para); flag_rule = flag_item_to_flag_rule(flag_item, u_para);
flag_item_free(flag_item); flag_item_free(flag_item);
@@ -341,7 +345,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
} }
char *key = (char *)&item_id; char *key = (char *)&item_id;
int ret = flag_runtime_update_row(flag_rt, key, sizeof(int), item_id, flag_rule, is_valid); int ret = flag_runtime_update_row(flag_rt, key, sizeof(uint64_t), item_id, flag_rule, is_valid);
if (ret < 0) { if (ret < 0) {
if (flag_rule != NULL) { if (flag_rule != NULL) {
flag_rule_free(flag_rule); flag_rule_free(flag_rule);
@@ -411,8 +415,7 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name)
} }
int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
uint64_t flag, int *group_ids, int hit_group_index, uint64_t flag, int vtable_id, struct maat_state *state)
size_t group_ids_size, int vtable_id, struct maat_state *state)
{ {
struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
@@ -429,8 +432,8 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
} }
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; uint64_t hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
memset(hit_item_ids, -1, sizeof(hit_item_ids)); memset(hit_item_ids, 0, sizeof(hit_item_ids));
for (int i = 0; i < n_hit_item; i++) { for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].rule_id; hit_item_ids[i] = hit_results[i].rule_id;
@@ -438,8 +441,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
size_t group_hit_cnt = 0; size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids, int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, hit_group_index, group_ids_size, n_hit_item, &group_hit_cnt, state);
&group_hit_cnt, state);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }

13
src/maat_fqdn_plugin.c
View File

@@ -35,8 +35,9 @@ struct fqdn_plugin_schema {
}; };
struct fqdn_plugin_item { struct fqdn_plugin_item {
int item_id; uint64_t item_id;
int suffix_flag; int suffix_flag;
}; };
struct fqdn_plugin_runtime { struct fqdn_plugin_runtime {
@@ -257,7 +258,7 @@ fqdn_plugin_item_new(const char *line, struct fqdn_plugin_schema *schema,
schema->table_id, line); schema->table_id, line);
goto error; goto error;
} }
item->item_id = atoi(line + column_offset); item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, schema->suffix_flag_column, &column_offset, &column_len); ret = get_column_pos(line, schema->suffix_flag_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -323,7 +324,11 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche
struct FQDN_rule *rule = NULL; struct FQDN_rule *rule = NULL;
struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema; struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema;
struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime;
int item_id = get_column_value(line, schema->item_id_column); long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
return -1;
}
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
@@ -344,7 +349,7 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche
char *key = (char *)&item_id; char *key = (char *)&item_id;
int ret = fqdn_plugin_runtime_update_row(fqdn_plugin_rt, schema, line, key, int ret = fqdn_plugin_runtime_update_row(fqdn_plugin_rt, schema, line, key,
sizeof(int), rule, is_valid); sizeof(uint64_t), rule, is_valid);
if (ret < 0) { if (ret < 0) {
if (item != NULL) { if (item != NULL) {
FREE(item); FREE(item);

139
src/maat_group.c
View File

@@ -21,8 +21,8 @@
#define MODULE_GROUP module_name_str("maat.group") #define MODULE_GROUP module_name_str("maat.group")
struct group2group_item { struct group2group_item {
int group_id; uint64_t group_id;
int super_group_id; uint64_t super_group_id;
}; };
struct group2group_schema { struct group2group_schema {
@@ -34,13 +34,13 @@ struct group2group_schema {
struct maat_group { struct maat_group {
igraph_integer_t vertex_id; igraph_integer_t vertex_id;
int group_id; uint64_t group_id;
int ref_by_compile_cnt; int ref_by_compile_cnt;
int ref_by_super_group_cnt; int ref_by_super_group_cnt;
int ref_by_sub_group_cnt; int ref_by_sub_group_cnt;
size_t top_group_cnt; size_t top_group_cnt;
int *top_group_ids; uint64_t *top_group_ids;
UT_hash_handle hh_group_id; UT_hash_handle hh_group_id;
UT_hash_handle hh_vertex_id; UT_hash_handle hh_vertex_id;
@@ -54,7 +54,6 @@ struct maat_group_topology {
igraph_vector_t dfs_vids; igraph_vector_t dfs_vids;
igraph_integer_t grp_vertex_id_generator; igraph_integer_t grp_vertex_id_generator;
pthread_rwlock_t rwlock;
struct log_handle *logger; struct log_handle *logger;
}; };
@@ -64,6 +63,7 @@ struct group2group_runtime {
uint32_t rule_num; uint32_t rule_num;
uint32_t updating_rule_num; uint32_t updating_rule_num;
pthread_rwlock_t rwlock;
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger; struct log_handle *logger;
}; };
@@ -128,9 +128,6 @@ struct maat_group_topology *maat_group_topology_new(struct log_handle *logger)
ret = igraph_empty(&group_topo->group_graph, 0, IGRAPH_DIRECTED); ret = igraph_empty(&group_topo->group_graph, 0, IGRAPH_DIRECTED);
assert(ret == IGRAPH_SUCCESS); assert(ret == IGRAPH_SUCCESS);
ret = pthread_rwlock_init(&group_topo->rwlock, NULL);
assert(ret == 0);
group_topo->logger = logger; group_topo->logger = logger;
return group_topo; return group_topo;
@@ -145,6 +142,7 @@ void *group2group_runtime_new(void *g2g_schema, int max_thread_num,
g2g_rt->group_topo = maat_group_topology_new(logger); g2g_rt->group_topo = maat_group_topology_new(logger);
g2g_rt->ref_garbage_bin = garbage_bin; g2g_rt->ref_garbage_bin = garbage_bin;
g2g_rt->logger = logger; g2g_rt->logger = logger;
pthread_rwlock_init(&g2g_rt->rwlock, NULL);
return g2g_rt; return g2g_rt;
} }
@@ -167,9 +165,6 @@ void maat_group_topology_free(struct maat_group_topology *group_topo)
assert(group_topo->hash_group_by_id == NULL); assert(group_topo->hash_group_by_id == NULL);
igraph_destroy(&group_topo->group_graph); igraph_destroy(&group_topo->group_graph);
pthread_rwlock_unlock(&group_topo->rwlock);
pthread_rwlock_destroy(&group_topo->rwlock);
} }
void group2group_runtime_free(void *g2g_runtime) void group2group_runtime_free(void *g2g_runtime)
@@ -184,6 +179,8 @@ void group2group_runtime_free(void *g2g_runtime)
maat_group_topology_free(g2g_rt->group_topo); maat_group_topology_free(g2g_rt->group_topo);
} }
//pthread_rwlock_unlock(&g2g_rt->rwlock);
pthread_rwlock_destroy(&g2g_rt->rwlock);
FREE(g2g_rt); FREE(g2g_rt);
} }
@@ -212,7 +209,7 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
g2g_schema->table_id, line); g2g_schema->table_id, line);
goto error; goto error;
} }
g2g_item->group_id = atoi(line + column_offset); g2g_item->group_id = atoll(line + column_offset);
ret = get_column_pos(line, g2g_schema->super_group_id_column, ret = get_column_pos(line, g2g_schema->super_group_id_column,
&column_offset, &column_len); &column_offset, &column_len);
@@ -222,7 +219,7 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
g2g_schema->table_id, line); g2g_schema->table_id, line);
goto error; goto error;
} }
g2g_item->super_group_id = atoi(line + column_offset); g2g_item->super_group_id = atoll(line + column_offset);
return g2g_item; return g2g_item;
error: error:
@@ -246,18 +243,18 @@ size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) {
return printed; return printed;
} }
struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id) struct maat_group *_group2group_runtime_add_group(void *g2g_runtime, uint64_t group_id, int lock_flag)
{ {
if (NULL == g2g_runtime) { struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
return NULL;
if (1 == lock_flag) {
pthread_rwlock_wrlock(&(g2g_rt->rwlock));
} }
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
struct maat_group_topology *group_topo = g2g_rt->group_topo; struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL); assert(group_topo != NULL);
struct maat_group *group = ALLOC(struct maat_group, 1); struct maat_group *group = ALLOC(struct maat_group, 1);
group->group_id = group_id; group->group_id = group_id;
group->vertex_id = group_topo->grp_vertex_id_generator++; group->vertex_id = group_topo->grp_vertex_id_generator++;
@@ -266,19 +263,32 @@ struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id
HASH_ADD(hh_group_id, group_topo->hash_group_by_id, group_id, sizeof(group->group_id), group); HASH_ADD(hh_group_id, group_topo->hash_group_by_id, group_id, sizeof(group->group_id), group);
HASH_ADD(hh_vertex_id, group_topo->hash_group_by_vertex, vertex_id, sizeof(group->vertex_id), group); HASH_ADD(hh_vertex_id, group_topo->hash_group_by_vertex, vertex_id, sizeof(group->vertex_id), group);
if (1 == lock_flag) {
pthread_rwlock_unlock(&(g2g_rt->rwlock));
}
return group; return group;
} }
void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group) struct maat_group *group2group_runtime_add_group(void *g2g_runtime, uint64_t group_id)
{ {
if (NULL == g2g_runtime || NULL == group) { if (NULL == g2g_runtime) {
return; return NULL;
} }
igraph_vector_t v; return _group2group_runtime_add_group(g2g_runtime, group_id, 1);
}
void _group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group, int lock_flag)
{
igraph_vector_t v;
char buff[4096] = {0}; char buff[4096] = {0};
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
if (1 == lock_flag) {
pthread_rwlock_wrlock(&(g2g_rt->rwlock));
}
struct maat_group_topology *group_topo = g2g_rt->group_topo; struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL); assert(group_topo != NULL);
@@ -302,26 +312,55 @@ void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *grou
HASH_DELETE(hh_vertex_id, group_topo->hash_group_by_vertex, group); HASH_DELETE(hh_vertex_id, group_topo->hash_group_by_vertex, group);
group_vertex_free(group); group_vertex_free(group);
if (1 == lock_flag) {
pthread_rwlock_unlock(&(g2g_rt->rwlock));
}
} }
struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id) void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group)
{
if (NULL == g2g_runtime || NULL == group) {
return;
}
_group2group_runtime_remove_group(g2g_runtime, group, 1);
}
struct maat_group *_group2group_runtime_find_group(void *g2g_runtime, uint64_t group_id, int lock_flag)
{ {
if (NULL == g2g_runtime) { if (NULL == g2g_runtime) {
return NULL; return NULL;
} }
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
if (1 == lock_flag) {
pthread_rwlock_rdlock(&(g2g_rt->rwlock));
}
struct maat_group_topology *group_topo = g2g_rt->group_topo; struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL); assert(group_topo != NULL);
struct maat_group *group = NULL; struct maat_group *group = NULL;
HASH_FIND(hh_group_id, group_topo->hash_group_by_id, &group_id, sizeof(group_id), group); HASH_FIND(hh_group_id, group_topo->hash_group_by_id, &group_id, sizeof(group_id), group);
if (1 == lock_flag) {
pthread_rwlock_unlock(&(g2g_rt->rwlock));
}
return group; return group;
} }
int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, struct maat_group *group2group_runtime_find_group(void *g2g_runtime, uint64_t group_id)
int super_group_id) {
if (NULL == g2g_runtime) {
return NULL;
}
return _group2group_runtime_find_group(g2g_runtime, group_id, 1);
}
int group2group_runtime_add_group_to_group(void *g2g_runtime, uint64_t group_id,
uint64_t super_group_id)
{ {
if (NULL == g2g_runtime) { if (NULL == g2g_runtime) {
return -1; return -1;
@@ -330,18 +369,20 @@ int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id,
int ret = 0; int ret = 0;
igraph_integer_t edge_id; igraph_integer_t edge_id;
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
pthread_rwlock_wrlock(&(g2g_rt->rwlock));
struct maat_group_topology *group_topo = g2g_rt->group_topo; struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL); assert(group_topo != NULL);
struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_id); struct maat_group *group = _group2group_runtime_find_group(g2g_runtime, group_id, 0);
if (NULL == group) { if (NULL == group) {
group = group2group_runtime_add_group(g2g_runtime, group_id); group = _group2group_runtime_add_group(g2g_runtime, group_id, 0);
} }
struct maat_group *super_group = group2group_runtime_find_group(g2g_runtime, struct maat_group *super_group = _group2group_runtime_find_group(g2g_runtime,
super_group_id); super_group_id, 0);
if (NULL == super_group) { if (NULL == super_group) {
super_group = group2group_runtime_add_group(g2g_runtime, super_group_id); super_group = _group2group_runtime_add_group(g2g_runtime, super_group_id, 0);
} }
ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id,
@@ -361,11 +402,12 @@ int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id,
ret = 0; ret = 0;
} }
pthread_rwlock_unlock(&(g2g_rt->rwlock));
return ret; return ret;
} }
int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group2group_runtime_remove_group_from_group(void *g2g_runtime, uint64_t group_id,
int group_id, int super_group_id) uint64_t super_group_id)
{ {
if (NULL == g2g_runtime) { if (NULL == g2g_runtime) {
return -1; return -1;
@@ -373,21 +415,24 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime,
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
pthread_rwlock_wrlock(&(g2g_rt->rwlock));
//No hash write operation, LOCK protection is unnecessary. //No hash write operation, LOCK protection is unnecessary.
struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_id); struct maat_group *group = _group2group_runtime_find_group(g2g_runtime, group_id, 0);
if (NULL == group) { if (NULL == group) {
log_error(g2g_rt->logger, MODULE_GROUP, log_error(g2g_rt->logger, MODULE_GROUP,
"Del group %d from group %d failed, group %d not exisited.", "Del group %d from group %d failed, group %d not exisited.",
group_id, super_group_id, group_id); group_id, super_group_id, group_id);
pthread_rwlock_unlock(&(g2g_rt->rwlock));
return -1; return -1;
} }
struct maat_group *super_group = group2group_runtime_find_group(g2g_runtime, struct maat_group *super_group = _group2group_runtime_find_group(g2g_runtime,
super_group_id); super_group_id, 0);
if (NULL == super_group) { if (NULL == super_group) {
log_error(g2g_rt->logger, MODULE_GROUP, log_error(g2g_rt->logger, MODULE_GROUP,
"Del group %d from group %d failed, superior group %d not exisited.", "Del group %d from group %d failed, superior group %d not exisited.",
group_id, super_group_id, super_group_id); group_id, super_group_id, super_group_id);
pthread_rwlock_unlock(&(g2g_rt->rwlock));
return -1; return -1;
} }
@@ -412,12 +457,14 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime,
igraph_es_destroy(&es); igraph_es_destroy(&es);
if (ret != IGRAPH_SUCCESS || edge_num_before - edge_num_after != 1) { if (ret != IGRAPH_SUCCESS || edge_num_before - edge_num_after != 1) {
pthread_rwlock_unlock(&(g2g_rt->rwlock));
assert(0); assert(0);
return -1; return -1;
} }
group->ref_by_super_group_cnt--; group->ref_by_super_group_cnt--;
super_group->ref_by_sub_group_cnt--; super_group->ref_by_sub_group_cnt--;
pthread_rwlock_unlock(&(g2g_rt->rwlock));
return 0; return 0;
} }
@@ -447,8 +494,10 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
struct maat_group *super_group = NULL; struct maat_group *super_group = NULL;
int tmp_vid=0; int tmp_vid=0;
size_t top_group_cnt=0; size_t top_group_cnt=0;
int* temp_group_ids=NULL; int *temp_group_ids=NULL;
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
pthread_rwlock_wrlock(&(g2g_rt->rwlock));
struct maat_group_topology *group_topo = g2g_rt->group_topo; struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL); assert(group_topo != NULL);
@@ -456,6 +505,7 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
igraph_is_dag(&(group_topo->group_graph), &is_dag); igraph_is_dag(&(group_topo->group_graph), &is_dag);
if (!is_dag) { if (!is_dag) {
log_error(g2g_rt->logger, MODULE_GROUP, "Sub group cycle detected!"); log_error(g2g_rt->logger, MODULE_GROUP, "Sub group cycle detected!");
pthread_rwlock_unlock(&(g2g_rt->rwlock));
return -1; return -1;
} }
@@ -472,7 +522,7 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
&& 0 == group->ref_by_sub_group_cnt) { && 0 == group->ref_by_sub_group_cnt) {
FREE(group->top_group_ids); FREE(group->top_group_ids);
group2group_runtime_remove_group(g2g_runtime, group); _group2group_runtime_remove_group(g2g_runtime, group, 0);
continue; continue;
} }
@@ -510,12 +560,13 @@ int group2group_runtime_build_top_groups(void *g2g_runtime)
free(group->top_group_ids); free(group->top_group_ids);
group->top_group_cnt = top_group_cnt; group->top_group_cnt = top_group_cnt;
group->top_group_ids = ALLOC(int, group->top_group_cnt); group->top_group_ids = ALLOC(uint64_t, group->top_group_cnt);
memcpy(group->top_group_ids, temp_group_ids, sizeof(int)*group->top_group_cnt); memcpy(group->top_group_ids, temp_group_ids, sizeof(uint64_t)*group->top_group_cnt);
FREE(temp_group_ids); FREE(temp_group_ids);
} }
igraph_vector_destroy(&group_topo->dfs_vids); igraph_vector_destroy(&group_topo->dfs_vids);
pthread_rwlock_unlock(&(g2g_rt->rwlock));
return 0; return 0;
} }
@@ -571,16 +622,19 @@ int group2group_runtime_commit(void *g2g_runtime, const char *table_name)
return ret; return ret;
} }
int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, int group2group_runtime_get_top_groups(void *g2g_runtime, uint64_t *group_ids,
size_t n_group_ids, int *top_group_ids) size_t n_group_ids, uint64_t *top_group_ids)
{ {
if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) { if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) {
return -1; return -1;
} }
size_t top_group_index = 0; size_t top_group_index = 0;
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
pthread_rwlock_rdlock(&(g2g_rt->rwlock));
for (size_t i = 0; i < n_group_ids; i++) { for (size_t i = 0; i < n_group_ids; i++) {
struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_ids[i]); struct maat_group *group = _group2group_runtime_find_group(g2g_runtime, group_ids[i], 0);
if (!group) { if (!group) {
continue; continue;
} }
@@ -589,6 +643,7 @@ int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids,
top_group_ids[top_group_index++] = group->top_group_ids[j]; top_group_ids[top_group_index++] = group->top_group_ids[j];
} }
} }
pthread_rwlock_unlock(&(g2g_rt->rwlock));
return top_group_index; return top_group_index;
} }

42
src/maat_interval.c
View File

@@ -34,8 +34,8 @@ struct interval_schema {
}; };
struct interval_item { struct interval_item {
int item_id; uint64_t item_id;
int group_id; uint64_t group_id;
int district_id; int district_id;
int low_bound; int low_bound;
int up_bound; int up_bound;
@@ -197,7 +197,7 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema
schema->table_id, line); schema->table_id, line);
goto error; goto error;
} }
item->item_id = atoi(line + column_offset); item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len); ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -206,7 +206,7 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema
schema->table_id, line); schema->table_id, line);
goto error; goto error;
} }
item->group_id = atoi(line + column_offset); item->group_id = atoll(line + column_offset);
table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id); table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id);
if (table_type == TABLE_TYPE_INTERVAL_PLUS) { if (table_type == TABLE_TYPE_INTERVAL_PLUS) {
@@ -276,7 +276,7 @@ void interval_rule_free(struct interval_rule *rule)
} }
int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, size_t key_len, int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, size_t key_len,
int item_id, struct interval_rule *rule, int is_valid) uint64_t item_id, struct interval_rule *rule, int is_valid)
{ {
void *data = NULL; void *data = NULL;
@@ -285,7 +285,7 @@ int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key,
data = rcu_hash_find(interval_rt->htable, key, key_len); data = rcu_hash_find(interval_rt->htable, key, key_len);
if (NULL == data) { if (NULL == data) {
log_error(interval_rt->logger, MODULE_INTERVAL, log_error(interval_rt->logger, MODULE_INTERVAL,
"the key of interval rule not exist, can't be deleted, item_id:%d", "the key of interval rule not exist, can't be deleted, item_id:%llu",
item_id); item_id);
return -1; return -1;
} }
@@ -295,7 +295,7 @@ int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key,
data = rcu_hash_find(interval_rt->htable, key, key_len); data = rcu_hash_find(interval_rt->htable, key, key_len);
if (data != NULL) { if (data != NULL) {
log_error(interval_rt->logger, MODULE_INTERVAL, log_error(interval_rt->logger, MODULE_INTERVAL,
"the key of interval rule already exist, can't be added, item_id:%d", "the key of interval rule already exist, can't be added, item_id:%llu",
item_id); item_id);
return -1; return -1;
} }
@@ -319,13 +319,17 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
struct interval_schema *schema = (struct interval_schema *)interval_schema; struct interval_schema *schema = (struct interval_schema *)interval_schema;
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
int item_id = get_column_value(line, schema->item_id_column); long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
return -1;
}
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
} else if (0 == is_valid) { } else if (0 == is_valid) {
//delete //delete
HASH_FIND_INT(interval_rt->item_hash, &item_id, item); HASH_FIND(hh, interval_rt->item_hash, &item_id, sizeof(uint64_t), item);
if (NULL == item) { if (NULL == item) {
return -1; return -1;
} }
@@ -341,10 +345,10 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
maat_garbage_bagging(interval_rt->ref_garbage_bin, u_para, maat_item_inner_free); maat_garbage_bagging(interval_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else { } else {
//add //add
HASH_FIND_INT(interval_rt->item_hash, &item_id, item); HASH_FIND(hh, interval_rt->item_hash, &item_id, sizeof(uint64_t), item);
if (item) { if (item) {
log_error(interval_rt->logger, MODULE_INTERVAL, log_error(interval_rt->logger, MODULE_INTERVAL,
"interval runtime add item %d to item_hash failed, already exist", "interval runtime add item %llu to item_hash failed, already exist",
item_id); item_id);
return -1; return -1;
} }
@@ -356,20 +360,20 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
u_para = maat_item_inner_new(interval_item->group_id, item_id, interval_item->district_id); u_para = maat_item_inner_new(interval_item->group_id, item_id, interval_item->district_id);
item = maat_item_new(item_id, interval_item->group_id, u_para); item = maat_item_new(item_id, interval_item->group_id, u_para);
HASH_ADD_INT(interval_rt->item_hash, item_id, item); HASH_ADD(hh, interval_rt->item_hash, item_id, sizeof(uint64_t), item);
interval_rule = interval_item_to_interval_rule(interval_item, u_para); interval_rule = interval_item_to_interval_rule(interval_item, u_para);
interval_item_free(interval_item); interval_item_free(interval_item);
if (NULL == interval_rule) { if (NULL == interval_rule) {
log_error(interval_rt->logger, MODULE_INTERVAL, log_error(interval_rt->logger, MODULE_INTERVAL,
"transform interval table(table_id:%d) item to interval_rule failed, item_id:%d", "transform interval table(table_id:%d) item to interval_rule failed, item_id:%llu",
schema->table_id, item_id); schema->table_id, item_id);
return -1; return -1;
} }
} }
char *key = (char *)&item_id; char *key = (char *)&item_id;
int ret = interval_runtime_update_row(interval_rt, key, sizeof(int), item_id, interval_rule, is_valid); int ret = interval_runtime_update_row(interval_rt, key, sizeof(uint64_t), item_id, interval_rule, is_valid);
if (ret < 0) { if (ret < 0) {
if (interval_rule != NULL) { if (interval_rule != NULL) {
interval_rule_free(interval_rule); interval_rule_free(interval_rule);
@@ -440,8 +444,7 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name)
} }
int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
uint64_t integer, int *group_ids, int hit_group_index, uint64_t integer, int vtable_id, struct maat_state *state)
size_t group_ids_size, int vtable_id, struct maat_state *state)
{ {
struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
@@ -458,12 +461,12 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
} }
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; uint64_t hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL; struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0; int real_hit_item_cnt = 0;
int district_id = state->district_id; int district_id = state->district_id;
memset(hit_item_ids, -1, sizeof(hit_item_ids)); memset(hit_item_ids, 0, sizeof(hit_item_ids));
for (int i = 0; i < n_hit_item; i++) { for (int i = 0; i < n_hit_item; i++) {
item = (struct maat_item_inner *)(hit_results[i].user_tag); item = (struct maat_item_inner *)(hit_results[i].user_tag);
@@ -474,8 +477,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
size_t group_hit_cnt = 0; size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids, int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, hit_group_index, group_ids_size, n_hit_item, &group_hit_cnt, state);
&group_hit_cnt, state);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }

33
src/maat_ip.c
View File

@@ -45,8 +45,8 @@ struct ipv6_item_rule {
}; };
struct ip_item { struct ip_item {
int item_id; uint64_t item_id;
int group_id; uint64_t group_id;
int addr_type; int addr_type;
union { union {
struct ipv4_item_rule ipv4; struct ipv4_item_rule ipv4;
@@ -218,7 +218,7 @@ struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema,
ip_schema->table_id, line); ip_schema->table_id, line);
goto error; goto error;
} }
ip_item->item_id = atoi(line + column_offset); ip_item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, ip_schema->group_id_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -227,7 +227,7 @@ struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema,
ip_schema->table_id, line); ip_schema->table_id, line);
goto error; goto error;
} }
ip_item->group_id = atoi(line + column_offset); ip_item->group_id = atoll(line + column_offset);
ret = get_column_pos(line, ip_schema->addr_type_column, &column_offset, &column_len); ret = get_column_pos(line, ip_schema->addr_type_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -365,13 +365,17 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
struct maat_item_inner *u_para = NULL; struct maat_item_inner *u_para = NULL;
struct ip_schema *schema = (struct ip_schema *)ip_schema; struct ip_schema *schema = (struct ip_schema *)ip_schema;
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
int item_id = get_column_value(line, schema->item_id_column); long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
return -1;
}
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
} else if (0 == is_valid) { } else if (0 == is_valid) {
//delete //delete
HASH_FIND_INT(ip_rt->item_hash, &item_id, item); HASH_FIND(hh, ip_rt->item_hash, &item_id, sizeof(uint64_t), item);
if (NULL == item) { if (NULL == item) {
return -1; return -1;
} }
@@ -388,10 +392,10 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
(void (*)(void *))maat_item_inner_free); (void (*)(void *))maat_item_inner_free);
} else { } else {
//add //add
HASH_FIND_INT(ip_rt->item_hash, &item_id, item); HASH_FIND(hh, ip_rt->item_hash, &item_id, sizeof(uint64_t), item);
if (item) { if (item) {
log_error(ip_rt->logger, MODULE_IP, log_error(ip_rt->logger, MODULE_IP,
"ip runtime add item %d to item_hash failed, already exist", "ip runtime add item %llu to item_hash failed, already exist",
item_id); item_id);
return -1; return -1;
} }
@@ -403,11 +407,11 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
u_para = maat_item_inner_new(ip_item->group_id, item_id, 0); u_para = maat_item_inner_new(ip_item->group_id, item_id, 0);
item = maat_item_new(item_id, ip_item->group_id, u_para); item = maat_item_new(item_id, ip_item->group_id, u_para);
HASH_ADD_INT(ip_rt->item_hash, item_id, item); HASH_ADD(hh, ip_rt->item_hash, item_id, sizeof(uint64_t), item);
} }
char *key = (char *)&item_id; char *key = (char *)&item_id;
int ret = ip_runtime_update_row(ip_rt, key, sizeof(int), ip_item, is_valid); int ret = ip_runtime_update_row(ip_rt, key, sizeof(uint64_t), ip_item, is_valid);
if (ret < 0) { if (ret < 0) {
if (ip_item != NULL) { if (ip_item != NULL) {
ip_item_free(ip_item); ip_item_free(ip_item);
@@ -477,8 +481,7 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name)
} }
int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
uint8_t *ip_addr, int *group_ids, int hit_group_index, uint8_t *ip_addr, int vtable_id, struct maat_state *state)
size_t group_id_size, int vtable_id, struct maat_state *state)
{ {
int n_hit_item = 0; int n_hit_item = 0;
struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
@@ -507,8 +510,8 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
} }
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; uint64_t hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
memset(hit_item_ids, -1, sizeof(hit_item_ids)); memset(hit_item_ids, 0, sizeof(hit_item_ids));
for (int i = 0; i < n_hit_item; i++) { for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = scan_results[i].rule_id; hit_item_ids[i] = scan_results[i].rule_id;
@@ -516,7 +519,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
size_t group_hit_cnt = 0; size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item, int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item,
group_ids, hit_group_index, group_id_size, &group_hit_cnt, state); &group_hit_cnt, state);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }

12
src/maat_ip_plugin.c
View File

@@ -33,7 +33,7 @@ struct ipv6_item_rule {
}; };
struct ip_plugin_item { struct ip_plugin_item {
int item_id; uint64_t item_id;
int ip_type; int ip_type;
union { union {
struct ipv4_item_rule ipv4; struct ipv4_item_rule ipv4;
@@ -226,7 +226,7 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
schema->table_id, line); schema->table_id, line);
goto error; goto error;
} }
ip_plugin_item->item_id = atoi(line + column_offset); ip_plugin_item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, schema->ip_type_column, &column_offset, &column_len); ret = get_column_pos(line, schema->ip_type_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
@@ -420,7 +420,11 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
struct ip_plugin_item *ip_plugin_item = NULL; struct ip_plugin_item *ip_plugin_item = NULL;
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
int item_id = get_column_value(line, schema->item_id_column); long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
return -1;
}
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
@@ -436,7 +440,7 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
} }
char *key = (char *)&item_id; char *key = (char *)&item_id;
int ret = ip_plugin_runtime_update_row(ip_plugin_rt, line, key, sizeof(int), int ret = ip_plugin_runtime_update_row(ip_plugin_rt, line, key, sizeof(uint64_t),
ip_plugin_item, is_valid); ip_plugin_item, is_valid);
if (ret < 0) { if (ret < 0) {
if (ip_plugin_item != NULL) { if (ip_plugin_item != NULL) {

19
src/maat_plugin.c
View File

@@ -362,9 +362,9 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
return -1; return -1;
} }
int item_id = get_column_value(line, schema->key_column); long long item_id = get_column_value(line, schema->key_column);
char *key = (char *)&item_id; char *key = (char *)&item_id;
ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid); ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(uint64_t), is_valid);
if (ret < 0) { if (ret < 0) {
schema->update_err_cnt++; schema->update_err_cnt++;
return -1; return -1;
@@ -423,3 +423,18 @@ const char *plugin_runtime_cached_row_get(void *plugin_runtime, size_t index)
return ex_data_runtime_cached_row_get(plugin_rt->ex_data_rt, index); return ex_data_runtime_cached_row_get(plugin_rt->ex_data_rt, index);
} }
void *plugin_runtime_get_ex_data(void *plugin_runtime, void *plugin_schema, const char *key)
{
if (NULL == plugin_runtime || NULL == plugin_schema) {
return NULL;
}
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
if (NULL == schema->ex_schema) {
return NULL;
}
return ex_data_runtime_get_ex_data_by_key(plugin_rt->ex_data_rt, key, strlen(key));
}

4
src/maat_rule.c
View File

@@ -32,7 +32,7 @@
#define MODULE_MAAT_RULE module_name_str("maat.rule") #define MODULE_MAAT_RULE module_name_str("maat.rule")
struct maat_item *maat_item_new(int item_id, int group_id, void *user_data) struct maat_item *maat_item_new(uint64_t item_id, uint64_t group_id, void *user_data)
{ {
struct maat_item *item = NULL; struct maat_item *item = NULL;
item = ALLOC(struct maat_item, 1); item = ALLOC(struct maat_item, 1);
@@ -206,7 +206,7 @@ error:
return ret; return ret;
} }
struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id) struct maat_item_inner *maat_item_inner_new(uint64_t group_id, uint64_t item_id, int district_id)
{ {
struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1); struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1);
item->magic_num = ITEM_RULE_MAGIC; item->magic_num = ITEM_RULE_MAGIC;

4
src/maat_utils.c
View File

@@ -51,7 +51,7 @@ char *maat_strdup(const char *s)
return d; return d;
} }
int get_column_value(const char *line, int column_seq) long long get_column_value(const char *line, int column_seq)
{ {
size_t column_offset = 0; size_t column_offset = 0;
size_t column_len = 0; size_t column_len = 0;
@@ -61,7 +61,7 @@ int get_column_value(const char *line, int column_seq)
return -1; return -1;
} }
return atoi(line + column_offset); return atoll(line + column_offset);
} }
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len) int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len)

2
src/maat_virtual.c
View File

@@ -81,7 +81,7 @@ int virtual_table_get_id(void *virtual_schema)
size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]) size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[])
{ {
if (NULL == virtual_schema) { if (NULL == virtual_schema) {
return -1; return 0;
} }
struct virtual_schema *schema = (struct virtual_schema *)virtual_schema; struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;

186
test/maat_framework_gtest.cpp
View File

@@ -40,7 +40,7 @@ TEST_F(MaatFlagScan, basic) {
//compile_id:192 flag: 0000 0001 mask: 0000 0011 //compile_id:192 flag: 0000 0001 mask: 0000 0011
//scan_data: 0000 1001 or 0000 1101 should hit //scan_data: 0000 1001 or 0000 1101 should hit
uint64_t scan_data = 9; uint64_t scan_data = 9;
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data, results, int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data, results,
@@ -83,7 +83,7 @@ TEST_F(MaatFlagScan, withExprRegion) {
//compile_id:193 flag: 0000 0010 mask: 0000 0011 //compile_id:193 flag: 0000 0010 mask: 0000 0011
//scan_data: 0000 0010 or 0000 0100 should hit //scan_data: 0000 0010 or 0000 0100 should hit
uint64_t flag_scan_data = 2; uint64_t flag_scan_data = 2;
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
@@ -114,7 +114,7 @@ TEST_F(MaatFlagScan, hitMultiCompile) {
//compile_id:194 flag: 0001 0101 mask: 0001 1111 //compile_id:194 flag: 0001 0101 mask: 0001 1111
//scan_data: 0001 0101 should hit compile192 and compile194 //scan_data: 0001 0101 should hit compile192 and compile194
uint64_t flag_scan_data = 21; uint64_t flag_scan_data = 21;
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
@@ -148,7 +148,7 @@ TEST_F(MaatStringScan, Expr8) {
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_table_get_id(g_maat_instance, table_name); int table_id = maat_table_get_id(g_maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data),
@@ -166,7 +166,7 @@ TEST_F(MaatStringScan, Expr8) {
TEST_F(MaatStringScan, Regex) { TEST_F(MaatStringScan, Regex) {
int ret = 0; int ret = 0;
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *cookie = "Cookie: Txa123aheadBCAxd"; const char *cookie = "Cookie: Txa123aheadBCAxd";
@@ -203,7 +203,7 @@ TEST_F(MaatStringScan, Regex) {
} }
TEST_F(MaatStringScan, ExprPlus) { TEST_F(MaatStringScan, ExprPlus) {
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *region_name1 ="HTTP URL"; const char *region_name1 ="HTTP URL";
@@ -237,7 +237,7 @@ TEST_F(MaatStringScan, ExprPlus) {
//TODO: //TODO:
#if 0 #if 0
TEST_F(MaatStringScan, ShouldNotHitExprPlus) { TEST_F(MaatStringScan, ShouldNotHitExprPlus) {
int results[ARRAY] = {0}; uint64_t results[ARRAY] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *region_name = "tcp.payload"; const char *region_name = "tcp.payload";
@@ -268,7 +268,7 @@ TEST_F(MaatStringScan, ShouldNotHitExprPlus) {
} }
TEST_F(MaatStringScan, ExprPlusWithHex) { TEST_F(MaatStringScan, ExprPlusWithHex) {
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *scan_data1 = "text/html; charset=UTF-8"; const char *scan_data1 = "text/html; charset=UTF-8";
@@ -302,7 +302,7 @@ TEST_F(MaatStringScan, ExprPlusWithHex) {
TEST_F(MaatStringScan, ExprPlusWithOffset) TEST_F(MaatStringScan, ExprPlusWithOffset)
{ {
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *region_name = "Payload"; const char *region_name = "Payload";
@@ -355,7 +355,7 @@ TEST_F(MaatStringScan, dynamic_config) {
const char *table_name = "HTTP_URL_LITERAL"; const char *table_name = "HTTP_URL_LITERAL";
int table_id = maat_table_get_id(g_maat_instance, table_name); int table_id = maat_table_get_id(g_maat_instance, table_name);
char data[128] = "hello world"; char data[128] = "hello world";
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results,
@@ -458,7 +458,7 @@ TEST_F(MaatIPScan, IPv4) {
int ret = inet_pton(AF_INET, ip_str, &sip); int ret = inet_pton(AF_INET, ip_str, &sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
int results[ARRAY_SIZE] = {-1}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE,
@@ -478,8 +478,7 @@ TEST_F(MaatIPScan, IPv6) {
int ret = inet_pton(AF_INET6, ip_str, &sip); int ret = inet_pton(AF_INET6, ip_str, &sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
int results[ARRAY_SIZE]; uint64_t results[ARRAY_SIZE] = {0};
memset(results, -1, sizeof(results));
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE,
@@ -499,7 +498,7 @@ TEST_F(MaatIPScan, dynamic_config) {
int ret = inet_pton(AF_INET, ip_str, &sip); int ret = inet_pton(AF_INET, ip_str, &sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
int results[ARRAY_SIZE] = {-1}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE,
@@ -589,7 +588,7 @@ protected:
}; };
TEST_F(MaatIntervalScan, Pure) { TEST_F(MaatIntervalScan, Pure) {
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *table_name = "CONTENT_SIZE"; const char *table_name = "CONTENT_SIZE";
@@ -610,7 +609,7 @@ TEST_F(MaatIntervalScan, Pure) {
} }
TEST_F(MaatIntervalScan, IntervalPlus) { TEST_F(MaatIntervalScan, IntervalPlus) {
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *table_name = "INTERGER_PLUS"; const char *table_name = "INTERGER_PLUS";
@@ -645,7 +644,7 @@ TEST_F(NOTLogic, ScanNotAtLast) {
const char *string_should_hit = "This string ONLY contains must-contained-string-of-rule-144."; const char *string_should_hit = "This string ONLY contains must-contained-string-of-rule-144.";
const char *string_should_not_hit = "This string contains both must-contained-string-of-rule-144 and must-not-contained-string-of-rule-144."; const char *string_should_not_hit = "This string contains both must-contained-string-of-rule-144 and must-not-contained-string-of-rule-144.";
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *hit_table_name = "HTTP_URL_LITERAL"; const char *hit_table_name = "HTTP_URL_LITERAL";
@@ -674,23 +673,19 @@ void maat_read_entry_start_cb(int update_type, void *u_para)
void maat_read_entry_cb(int table_id, const char *table_line, void *u_para) void maat_read_entry_cb(int table_id, const char *table_line, void *u_para)
{ {
char ip_str[16]={0}; char ip_str[16] = {0};
int entry_id=-1,seq=-1; int entry_id = -1, seq = -1;
unsigned int ip_uint=0; unsigned int ip_uint = 0;
int is_valid=0; int is_valid = 0;
unsigned int local_ip_nr=16820416;//192.168.0.1 unsigned int local_ip_nr = 16820416;//192.168.0.1
sscanf(table_line, "%d\t%s\t%d\t%d", &seq,ip_str, &entry_id, &is_valid); sscanf(table_line, "%d\t%s\t%d\t%d", &seq,ip_str, &entry_id, &is_valid);
inet_pton(AF_INET,ip_str,&ip_uint); inet_pton(AF_INET, ip_str, &ip_uint);
if(local_ip_nr==ip_uint) if (local_ip_nr == ip_uint) {
{ if (is_valid == 1) {
if(is_valid==1)
{
//printf("Load entry id %d success.\n",entry_id); //printf("Load entry id %d success.\n",entry_id);
EXPECT_EQ(entry_id, 101); EXPECT_EQ(entry_id, 101);
} } else {
else
{
//printf("Offload entry id %d success.\n",entry_id); //printf("Offload entry id %d success.\n",entry_id);
} }
} }
@@ -745,7 +740,7 @@ protected:
}; };
struct ip_plugin_ud { struct ip_plugin_ud {
int rule_id; uint64_t rule_id;
char *buffer; char *buffer;
int ref_cnt; int ref_cnt;
}; };
@@ -759,7 +754,7 @@ void ip_plugin_EX_new_cb(int table_id, const char *key, const char *table_line,
int ret = get_column_pos(table_line, 1, &column_offset, &column_len); int ret = get_column_pos(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
ud->rule_id = atoi(table_line + column_offset); ud->rule_id = atoll(table_line + column_offset);
ret = get_column_pos(table_line, 5, &column_offset, &column_len); ret = get_column_pos(table_line, 5, &column_offset, &column_len);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
@@ -853,7 +848,7 @@ protected:
}; };
TEST_F(VirtualTable, basic) { TEST_F(VirtualTable, basic) {
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *table_name = "HTTP_RESPONSE_KEYWORDS"; const char *table_name = "HTTP_RESPONSE_KEYWORDS";
@@ -930,7 +925,7 @@ void compile_ex_param_dup(int idx, void **to, void **from, long argl, void *argp
} }
TEST_F(CompileTable, CompileEXData) { TEST_F(CompileTable, CompileEXData) {
int results[ARRAY_SIZE] = {0}; uint64_t results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
@@ -964,6 +959,129 @@ TEST_F(CompileTable, CompileEXData) {
maat_state_free(&state); maat_state_free(&state);
} }
class MaatCmdTest : public testing::Test
{
protected:
static void SetUpTestCase() {
}
static void TearDownTestCase() {
}
};
struct user_info {
char name[256];
char ip_addr[32];
int id;
int ref_cnt;
};
void plugin_ex_new_cb(int table_id, const char *key, const char *table_line,
void **ad, long argl, void *argp)
{
int *counter = (int *)argp;
struct user_info *u = ALLOC(struct user_info, 1);
int valid = 0, tag = 0;
int ret = sscanf(table_line, "%d\t%s\t%s%d\t%d", &(u->id), u->ip_addr, u->name, &valid, &tag);
EXPECT_EQ(ret, 5);
u->ref_cnt = 1;
*ad = u;
(*counter)++;
}
void plugin_ex_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct user_info *u = (struct user_info *)(*ad);
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) {
free(u);
*ad = NULL;
}
}
void plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct user_info *u = (struct user_info *)(*from);
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
#if 0
TEST_F(MaatCmdTest, PluginEXData) {
const char *table_name = "TEST_PLUGIN_EXDATA_TABLE";
const int TEST_CMD_LINE_NUM = 4;
struct maat_cmd_line line_rule;
const char *table_line[TEST_CMD_LINE_NUM] = {"1\t192.168.0.1\tmahuateng\t1\t0",
"2\t192.168.0.2\tliuqiangdong\t1\t0",
"3\t192.168.0.3\tmayun\t1\t0",
"4\t192.168.0.4\tliyanhong\t1\t0"};
int table_id = maat_table_get_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
/* 1st line */
memset(&line_rule, 0, sizeof(line_rule));
line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1);
line_rule.table_name = table_name;
line_rule.table_line = table_line[0];
line_rule.expire_after = 0;
int ret = maat_cmd_set_line(g_maat_instance, &line_rule);
EXPECT_GT(ret, 0);
/* 2nd line */
memset(&line_rule, 0, sizeof(line_rule));
line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1);
line_rule.table_name = table_name;
line_rule.table_line = table_line[1];
line_rule.expire_after = 0;
ret = maat_cmd_set_line(g_maat_instance, &line_rule);
EXPECT_GT(ret, 0);
/* 3rd line */
memset(&line_rule, 0, sizeof(line_rule));
line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1);
line_rule.table_name = table_name;
line_rule.table_line = table_line[2];
line_rule.expire_after = 0;
ret = maat_cmd_set_line(g_maat_instance, &line_rule);
EXPECT_GT(ret, 0);
/* 4th line */
memset(&line_rule, 0, sizeof(line_rule));
line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1);
line_rule.table_name = table_name;
line_rule.table_line = table_line[3];
line_rule.expire_after = 0;
ret = maat_cmd_set_line(g_maat_instance, &line_rule);
EXPECT_GT(ret, 0);
sleep(1);
int ex_data_counter = 0;
ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id,
plugin_ex_new_cb,
plugin_ex_free_cb,
plugin_ex_dup_cb,
0, &ex_data_counter);
ASSERT_TRUE(ret >= 0);
EXPECT_EQ(ex_data_counter, TEST_CMD_LINE_NUM);
struct user_info *uinfo = NULL;
uinfo = (struct user_info *)maat_plugin_table_get_ex_data(g_maat_instance, table_id, "192.168.0.2");
ASSERT_TRUE(uinfo != NULL);
EXPECT_EQ(0, strcmp(uinfo->name, "liuqiangdong"));
EXPECT_EQ(uinfo->id, 2);
plugin_ex_free_cb(table_id, (void**)&uinfo, 0, NULL);
ret = maat_cmd_set_line(g_maat_instance, &line_rule + 1);
EXPECT_GT(ret, 0);
sleep(1);
uinfo = (struct user_info *)maat_plugin_table_get_ex_data(g_maat_instance, table_id, "192.168.0.2");
ASSERT_TRUE(uinfo == NULL);
}
#endif
int count_line_num_cb(const char *table_name, const char *line, void *u_para) int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{ {
(*((unsigned int *)u_para))++; (*((unsigned int *)u_para))++;

6
test/maat_input_mode_gtest.cpp
View File

@@ -39,7 +39,7 @@ TEST(json_mode, maat_scan_string) {
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_table_get_id(maat_instance, table_name); int table_id = maat_table_get_id(maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
int results[5] = {0}; uint64_t results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data),
@@ -84,7 +84,7 @@ TEST(iris_mode, maat_scan_string) {
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_table_get_id(maat_instance, table_name); int table_id = maat_table_get_id(maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
int results[5] = {0}; uint64_t results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data),
@@ -206,7 +206,7 @@ TEST(redis_mode, maat_scan_string) {
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_table_get_id(maat_instance, table_name); int table_id = maat_table_get_id(maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
int results[5] = {0}; uint64_t results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data),