gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support flag_plus table

Browse Source
This commit is contained in:
liuwentan
2023-03-01 17:44:07 +08:00
parent 2c6cca6f56
commit d429a6346d
16 changed files with 546 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
include/maat.h
View File

@@ -56,6 +56,20 @@ struct ip_addr {
};
};
struct ipv4_tuple {
unsigned int sip; /* network order */
unsigned int dip; /* network order */
unsigned short sport; /* network order */
unsigned short dport; /* network order */
};
struct ipv6_tuple {
unsigned int sip[4] ; /* network order */
unsigned int dip[4] ; /* network order */
unsigned short sport; /* network order */
unsigned short dport; /* network order */
};
typedef void maat_start_callback_t(int update_type, void *u_param);
typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para);
typedef void maat_finish_callback_t(void *u_para);
@@ -86,7 +100,7 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path);
void maat_free(struct maat *instance);
/* maat table API */
int maat_table_get_id(struct maat *instance, const char *table_name);
int maat_get_table_id(struct maat *instance, const char *table_name);
/* return 0 if success, otherwise return -1 */
int maat_table_callback_register(struct maat *instance, int table_id,
@@ -138,17 +152,27 @@ int maat_scan_flag(struct maat *instance, int table_id, int thread_id,
long long flag, long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
unsigned int intval, long long *results, size_t n_result,
long long integer, long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
int maat_scan_ipv4(struct maat *instance, int table_id, int thread_id,
uint32_t ip_addr, long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
int maat_scan_ipv4_tuple4(struct maat *instance, int table_id, int thread_id,
const struct ipv4_tuple *tuple, long long *results,
size_t n_result, size_t *n_hit_result,
struct maat_state **state);
int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id,
uint8_t *ip_addr, long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state);
int maat_scan_ipv6_tuple4(struct maat *instance, int table_id, int thread_id,
const struct ipv6_tuple *tuple, long long *results,
size_t n_result, size_t *n_hit_result,
struct maat_state **state);
int maat_scan_string(struct maat *instance, int table_id, int thread_id,
const char *data, size_t data_len, long long *results,
size_t n_result, size_t *n_hit_result,

5
src/inc_internal/maat_table.h
View File

@@ -25,13 +25,13 @@ extern "C"
enum table_type {
TABLE_TYPE_INVALID = -1,
TABLE_TYPE_FLAG = 0,
TABLE_TYPE_FLAG_PLUS,
TABLE_TYPE_EXPR,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_IP_PLUS,
TABLE_TYPE_PORT,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_INTERVAL_PLUS,
TABLE_TYPE_DIGEST,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_CONJUNCTION,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
@@ -39,6 +39,7 @@ enum table_type {
TABLE_TYPE_BOOL_PLUGIN,
//above are physical table
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_COMPOSITION,
TABLE_TYPE_COMPILE,
TABLE_TYPE_GROUP2GROUP,
TABLE_TYPE_GROUP2COMPILE,

104
src/json2iris.c
View File

@@ -139,7 +139,9 @@ int set_iris_descriptor(const char *json_file, cJSON *json,
maat_kv_register(iris_cfg->str2int_map, "no", 0);
maat_kv_register(iris_cfg->str2int_map, "flag", TABLE_TYPE_FLAG);
maat_kv_register(iris_cfg->str2int_map, "flag_plus", TABLE_TYPE_FLAG_PLUS);
maat_kv_register(iris_cfg->str2int_map, "ip_plus", TABLE_TYPE_IP_PLUS);
maat_kv_register(iris_cfg->str2int_map, "port", TABLE_TYPE_PORT);
maat_kv_register(iris_cfg->str2int_map, "string", TABLE_TYPE_EXPR);
maat_kv_register(iris_cfg->str2int_map, "expr", TABLE_TYPE_EXPR);
maat_kv_register(iris_cfg->str2int_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
@@ -147,8 +149,6 @@ int set_iris_descriptor(const char *json_file, cJSON *json,
maat_kv_register(iris_cfg->str2int_map, "interval", TABLE_TYPE_INTERVAL);
maat_kv_register(iris_cfg->str2int_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS);
maat_kv_register(iris_cfg->str2int_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS);
maat_kv_register(iris_cfg->str2int_map, "digest", TABLE_TYPE_DIGEST);
maat_kv_register(iris_cfg->str2int_map, "similar", TABLE_TYPE_SIMILARITY);
maat_kv_register(iris_cfg->str2int_map, "ipv4", 4);
maat_kv_register(iris_cfg->str2int_map, "ipv6", 6);
@@ -423,6 +423,12 @@ int write_flag_line(cJSON *region_json, struct iris_description *p_iris,
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
if (table->table_type==TABLE_TYPE_FLAG_PLUS) {
json_cmd[cmd_cnt].json_string = "district";
json_cmd[cmd_cnt].json_type = cJSON_String;
cmd_cnt++;
}
json_cmd[cmd_cnt].json_string = "flag";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
@@ -637,72 +643,6 @@ int write_intval_line(cJSON *region_json, struct iris_description *p_iris,
json_cmd, cmd_cnt, table, logger);
}
int write_digest_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
memset(json_cmd, 0, sizeof(json_cmd));
json_cmd[cmd_cnt].json_string = "region_id";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "group_id";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "raw_len";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "digest";
json_cmd[cmd_cnt].json_type = cJSON_String;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "cfds_level";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "is_valid";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
}
int write_similar_line(cJSON *region_json, struct iris_description *p_iris,
struct iris_table *table, struct log_handle *logger)
{
struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0;
memset(json_cmd, 0, sizeof(json_cmd));
json_cmd[cmd_cnt].json_string = "region_id";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "group_id";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "target";
json_cmd[cmd_cnt].json_type = cJSON_String;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "threshold";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
json_cmd[cmd_cnt].json_string = "is_valid";
json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map,
json_cmd, cmd_cnt, table, logger);
}
int write_region_rule(cJSON *region_json, int compile_id, int group_id,
struct iris_description *p_iris, struct log_handle *logger)
{
@@ -750,6 +690,7 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id,
switch(table_type)
{
case TABLE_TYPE_FLAG:
case TABLE_TYPE_FLAG_PLUS:
ret = write_flag_line(table_content, p_iris, table_info, logger);
break;
case TABLE_TYPE_EXPR:
@@ -763,12 +704,6 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id,
case TABLE_TYPE_INTERVAL_PLUS:
ret = write_intval_line(table_content, p_iris, table_info, logger);
break;
case TABLE_TYPE_DIGEST:
ret = write_digest_line(table_content, p_iris, table_info, logger);
break;
case TABLE_TYPE_SIMILARITY:
ret = write_similar_line(table_content, p_iris, table_info, logger);
break;
default:
assert(0);
break;
@@ -778,10 +713,16 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id,
int write_group2compile_line(int group_id, int compile_id, int group_not_flag,
int clause_index, const char *vtable,
struct iris_description *p_iris)
struct iris_description *p_iris,
struct iris_table *g2c_table)
{
char buff[4096] = {0};
struct iris_table *table = p_iris->group2compile_table;
struct iris_table *table = NULL;
if (g2c_table != NULL) {
table = g2c_table;
} else {
table = p_iris->group2compile_table;
}
snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id,
group_not_flag, vtable, clause_index);
@@ -818,6 +759,7 @@ int write_group_rule(cJSON *group_json, int parent_id,
const char *str_parent_type[2] = {"compile", "group"};
const char *group_name = NULL;
const char *virtual_table = NULL;
struct iris_table *g2c_table = NULL;
cJSON *item = cJSON_GetObjectItem(group_json, "group_name");
if (NULL == item || item->type != cJSON_String) {
@@ -847,6 +789,12 @@ int write_group_rule(cJSON *group_json, int parent_id,
} else {
clause_index = item->valueint;
}
item = cJSON_GetObjectItem(group_json, "g2c_table_name");
if (item != NULL && item->type == cJSON_String) {
g2c_table = query_table_info(p_iris, item->valuestring,
TABLE_TYPE_GROUP2COMPILE);
}
} else {
group_not_flag = 0;
}
@@ -892,7 +840,7 @@ int write_group_rule(cJSON *group_json, int parent_id,
if (parent_type == PARENT_TYPE_COMPILE) {
ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag,
clause_index, virtual_table, p_iris);
clause_index, virtual_table, p_iris, g2c_table);
} else {
ret = write_group2group_line(group_info->group_id, parent_id, p_iris);
}
@@ -998,7 +946,7 @@ int write_compile_line(cJSON *compile, struct iris_description *p_iris,
cmd_cnt++;
struct iris_table *table_info = NULL;
item = cJSON_GetObjectItem(compile,"table_name");
item = cJSON_GetObjectItem(compile,"compile_table_name");
if (NULL == item || item->type != cJSON_String) {
table_info = p_iris->compile_table;
} else {

53
src/maat_api.c
View File

@@ -61,6 +61,7 @@ enum scan_type maat_table_get_scan_type(enum table_type table_type)
switch (table_type) {
case TABLE_TYPE_FLAG:
case TABLE_TYPE_FLAG_PLUS:
ret = SCAN_TYPE_FLAG;
break;
case TABLE_TYPE_EXPR:
@@ -386,7 +387,7 @@ void maat_free(struct maat *maat_instance)
pthread_join(maat_instance->cfg_mon_thread, &ret);
}
int maat_table_get_id(struct maat *maat_instance, const char *table_name)
int maat_get_table_id(struct maat *maat_instance, const char *table_name)
{
int table_id = -1;
@@ -889,7 +890,12 @@ int flag_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long
for (int i = 0; i < physical_table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if (table_type != TABLE_TYPE_FLAG) {
if ((table_type == TABLE_TYPE_FLAG_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
return -1;
}
if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) {
continue;
}
@@ -913,7 +919,7 @@ int flag_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long
return sum_hit_group_cnt;
}
int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long long intval,
int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long long integer,
int physical_table_ids[], int physical_table_cnt, int vtable_id,
struct maat_state *mid)
{
@@ -924,7 +930,7 @@ int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id,
if ((table_type == TABLE_TYPE_INTERVAL_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
//maat_instance->scan_err_cnt++;
return MAAT_SCAN_ERR;
return -1;
}
if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) {
@@ -933,13 +939,13 @@ int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id,
void *interval_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]);
if (NULL == interval_rt) {
return MAAT_SCAN_ERR;
return -1;
}
int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt,
thread_id, intval, vtable_id, mid);
thread_id, integer, vtable_id, mid);
if (group_hit_cnt < 0) {
return MAAT_SCAN_ERR;
return -1;
}
if (group_hit_cnt > 0) {
@@ -965,13 +971,13 @@ int ipv4_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint
void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]);
if (NULL == ip_rt) {
return MAAT_SCAN_ERR;
return -1;
}
int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4,
(uint8_t *)&ip_addr, vtable_id, mid);
if (group_hit_cnt < 0) {
return MAAT_SCAN_ERR;
return -1;
}
if (group_hit_cnt > 0) {
@@ -997,13 +1003,13 @@ int ipv6_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint
void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]);
if (NULL == ip_rt) {
return MAAT_SCAN_ERR;
return -1;
}
int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6,
ip_addr, vtable_id, mid);
if (group_hit_cnt < 0) {
return MAAT_SCAN_ERR;
return -1;
}
if (group_hit_cnt > 0) {
@@ -1104,7 +1110,7 @@ size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n
compile_table_cnt = 1;
} else {
for (size_t i = 0; i < mid->n_compile_table; i++) {
compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]);
compile_table_id[i] = maat_get_table_id(maat_instance, mid->compile_tables[i]);
}
compile_table_cnt = mid->n_compile_table;
}
@@ -1158,7 +1164,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
enum table_type table_type = TABLE_TYPE_INVALID;
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_FLAG) {
if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) {
return MAAT_SCAN_ERR;
}
}
@@ -1200,7 +1206,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
}
int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id,
unsigned int intval, long long *results, size_t n_result,
long long integer, long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state)
{
if ((NULL == maat_instance) || table_id < 0 || table_id >= MAX_TABLE_NUM
@@ -1242,7 +1248,7 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id,
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = interval_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, intval,
int hit_group_cnt = interval_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, integer,
physical_table_ids,
physical_table_cnt, vtable_id, mid);
if (hit_group_cnt < 0) {
@@ -1351,6 +1357,13 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
return MAAT_SCAN_OK;
}
int maat_scan_ipv4_tuple4(struct maat *instance, int table_id, int thread_id,
const struct ipv4_tuple *tuple4, long long *results,
size_t n_result, size_t *n_hit_result,
struct maat_state **state)
{
return MAAT_SCAN_OK;
}
int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id,
uint8_t *ip_addr, long long *results, size_t n_result,
@@ -1428,6 +1441,14 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id,
return MAAT_SCAN_OK;
}
int maat_scan_ipv6_tuple4(struct maat *instance, int table_id, int thread_id,
const struct ipv6_tuple *tuple, long long *results,
size_t n_result, size_t *n_hit_result,
struct maat_state **state)
{
return MAAT_SCAN_OK;
}
int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
const char *data, size_t data_len, long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state)
@@ -1691,7 +1712,7 @@ size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *state,
compile_table_cnt = 1;
} else {
for (size_t i = 0; i < state->n_compile_table; i++) {
compile_table_ids[i] = maat_table_get_id(maat_instance, state->compile_tables[i]);
compile_table_ids[i] = maat_get_table_id(maat_instance, state->compile_tables[i]);
}
compile_table_cnt = state->n_compile_table;
}

16
src/maat_command.c
View File

@@ -163,9 +163,16 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
int column_seq = 0;
switch (table_type) {
case TABLE_TYPE_FLAG:
case TABLE_TYPE_FLAG_PLUS: //TODO:
column_seq = 5;
break;
case TABLE_TYPE_EXPR:
column_seq = 7;
break;
case TABLE_TYPE_EXPR_PLUS:
column_seq = 8;
break;
case TABLE_TYPE_IP_PLUS:
column_seq = 18;
break;
@@ -188,15 +195,6 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
case TABLE_TYPE_INTERVAL_PLUS:
column_seq = 6;
break;
case TABLE_TYPE_DIGEST:
column_seq = 6;
break;
case TABLE_TYPE_SIMILARITY:
column_seq = 5;
break;
case TABLE_TYPE_EXPR_PLUS:
column_seq = 8;
break;
case TABLE_TYPE_GROUP2COMPILE:
case TABLE_TYPE_GROUP2GROUP:
column_seq = 3;

2
src/maat_compile.c
View File

@@ -1807,7 +1807,7 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
compile_table_cnt = 1;
} else {
for (size_t i = 0; i < state->n_compile_table; i++) {
compile_table_ids[i] = maat_table_get_id(state->maat_instance, state->compile_tables[i]);
compile_table_ids[i] = maat_get_table_id(state->maat_instance, state->compile_tables[i]);
}
compile_table_cnt = state->n_compile_table;
}

75
src/maat_flag.c
View File

@@ -27,6 +27,7 @@
struct flag_schema {
int item_id_column;
int group_id_column;
int district_column;
int flag_column;
int flag_mask_column;
int table_id;
@@ -36,6 +37,7 @@ struct flag_schema {
struct flag_item {
long long item_id;
long long group_id;
int district_id;
long long flag;
long long flag_mask;
};
@@ -88,6 +90,12 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "district");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->district_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "flag");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->flag_column = custom_item->valueint;
@@ -205,53 +213,77 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_
return 0;
}
struct flag_item *flag_item_new(const char *line, struct flag_schema *flag_schema,
struct flag_item *flag_item_new(const char *line, struct flag_schema *schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
struct flag_item *flag_item = ALLOC(struct flag_item, 1);
enum table_type table_type = TABLE_TYPE_INVALID;
struct flag_item *item = ALLOC(struct flag_item, 1);
int ret = get_column_pos(line, flag_schema->item_id_column, &column_offset, &column_len);
int ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no item_id",
flag_schema->table_id, line);
schema->table_id, line);
goto error;
}
flag_item->item_id = atoll(line + column_offset);
item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, flag_schema->group_id_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no group_id",
flag_schema->table_id, line);
schema->table_id, line);
goto error;
}
flag_item->group_id = atoll(line + column_offset);
item->group_id = atoll(line + column_offset);
ret = get_column_pos(line, flag_schema->flag_column, &column_offset, &column_len);
table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id);
if (table_type == TABLE_TYPE_INTERVAL_PLUS) {
ret = get_column_pos(line, schema->district_column, &column_offset, &column_len);
if (ret < 0) {
goto error;
}
if (column_len >= MAX_DISTRICT_STR) {
log_error(logger, MODULE_FLAG,
"flag_plus table(table_id:%d) line:%s district length too long",
schema->table_id, line);
goto error;
}
char district[MAX_DISTRICT_STR] = {0};
memcpy(district, (line + column_offset), column_len);
assert(strlen(district) > 0);
str_unescape(district);
item->district_id = table_manager_get_district_id(schema->ref_tbl_mgr, district);
} else {
item->district_id = DISTRICT_ANY;
}
ret = get_column_pos(line, schema->flag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no flag",
flag_schema->table_id, line);
schema->table_id, line);
goto error;
}
flag_item->flag = strtoull(line + column_offset, NULL, 0);
item->flag = strtoull(line + column_offset, NULL, 0);
ret = get_column_pos(line, flag_schema->flag_mask_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->flag_mask_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FLAG,
"flag table(table_id:%d) line:%s has no flag_mask",
flag_schema->table_id, line);
schema->table_id, line);
goto error;
}
flag_item->flag_mask = strtoull(line + column_offset, NULL, 0);
item->flag_mask = strtoull(line + column_offset, NULL, 0);
return flag_item;
return item;
error:
FREE(flag_item);
FREE(item);
return NULL;
}
@@ -330,7 +362,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
return -1;
}
u_para = maat_item_inner_new(flag_item->group_id, item_id, 0);
u_para = maat_item_inner_new(flag_item->group_id, item_id, flag_item->district_id);
item = maat_item_new(item_id, flag_item->group_id, u_para);
HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(long long), item);
@@ -433,10 +465,17 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
}
long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
int district_id = state->district_id;
memset(hit_item_ids, 0, sizeof(hit_item_ids));
for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].rule_id;
item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (item->district_id == district_id || item->district_id == DISTRICT_ANY) {
hit_item_ids[real_hit_item_cnt++] = hit_results[i].rule_id;
}
}
size_t group_hit_cnt = 0;

2
src/maat_interval.c
View File

@@ -215,7 +215,7 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema
if (column_len >= MAX_DISTRICT_STR) {
log_error(logger, MODULE_INTERVAL,
"interval table(table_id:%d) line:%s district length too long",
"interval_plus table(table_id:%d) line:%s district length too long",
schema->table_id, line);
goto error;
}

12
src/maat_redis_monitor.c
View File

@@ -190,7 +190,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
for (i = 0; i < rule_num; i++) {
snprintf(redis_cmd, sizeof(redis_cmd),
"GET %s:%s,%lu", mr_key_prefix[rule_list[i].op],
"GET %s:%s,%lld", mr_key_prefix[rule_list[i].op],
rule_list[i].table_name,
rule_list[i].rule_id);
ret = redisAppendCommand(c, redis_cmd);
@@ -236,7 +236,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
for (i = 0; i < failed_cnt; i++) {
idx = retry_ids[i];
snprintf(redis_cmd, sizeof(redis_cmd),
"GET %s:%s,%lu", mr_key_prefix[MAAT_OP_DEL],
"GET %s:%s,%lld", mr_key_prefix[MAAT_OP_DEL],
rule_list[idx].table_name,
rule_list[idx].rule_id);
ret = redisAppendCommand(c, redis_cmd);
@@ -454,13 +454,13 @@ int recovery_history_version(const struct serial_rule *current, int current_num,
struct serial_rule *s_rule_map = NULL;
struct serial_rule *rule_node = NULL;
for (i = 0; i < current_num; i++) {
snprintf(hkey, sizeof(hkey), "%ld,%s", current[i].rule_id, current[i].table_name);
snprintf(hkey, sizeof(hkey), "%lld,%s", current[i].rule_id, current[i].table_name);
rule_node = serial_rule_clone(current + i);
HASH_ADD_KEYPTR(hh, s_rule_map, hkey, strlen(hkey), rule_node);
}
for (i = changed_num - 1; i >= 0; i--) {
snprintf(hkey, sizeof(hkey), "%ld,%s", changed[i].rule_id, changed[i].table_name);
snprintf(hkey, sizeof(hkey), "%lld,%s", changed[i].rule_id, changed[i].table_name);
//newly added rule is need to delete from current, so that history version can be recovered.
if (changed[i].op == MAAT_OP_ADD) {
rule_node = NULL;
@@ -643,7 +643,7 @@ FULL_UPDATE:
continue;
}
ret = sscanf(sub_reply->element[i]->str, "%*[^:]:%[^,],%ld",
ret = sscanf(sub_reply->element[i]->str, "%*[^:]:%[^,],%lld",
s_rule_array[full_idx].table_name,
&(s_rule_array[full_idx].rule_id));
s_rule_array[full_idx].op = MAAT_OP_ADD;
@@ -1316,7 +1316,7 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger)
for (size_t i = 0; i < s_rule_num; i++) {
s_rule[i].op = MAAT_OP_DEL;
ret = sscanf(data_reply->element[i]->str, "%[^,],%ld",
ret = sscanf(data_reply->element[i]->str, "%[^,],%lld",
s_rule[i].table_name, &(s_rule[i].rule_id));
assert(ret == 2);
}

36
src/maat_table.c
View File

@@ -86,6 +86,15 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.update_runtime = flag_runtime_update,
.commit_runtime = flag_runtime_commit
},
{
.type = TABLE_TYPE_FLAG_PLUS,
.new_schema = flag_schema_new,
.free_schema = flag_schema_free,
.new_runtime = flag_runtime_new,
.free_runtime = flag_runtime_free,
.update_runtime = flag_runtime_update,
.commit_runtime = flag_runtime_commit
},
{
.type = TABLE_TYPE_EXPR,
.new_schema = expr_schema_new,
@@ -113,6 +122,15 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.update_runtime = ip_runtime_update,
.commit_runtime = ip_runtime_commit
},
{
.type = TABLE_TYPE_PORT,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_INTERVAL,
.new_schema = interval_schema_new,
@@ -131,12 +149,6 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.update_runtime = interval_runtime_update,
.commit_runtime = interval_runtime_commit
},
{
.type = TABLE_TYPE_DIGEST
},
{
.type = TABLE_TYPE_SIMILARITY
},
{
.type = TABLE_TYPE_CONJUNCTION,
.new_schema = NULL,
@@ -191,6 +203,15 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_COMPOSITION,
.new_schema = NULL,
.free_schema = NULL,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = NULL,
.commit_runtime = NULL
},
{
.type = TABLE_TYPE_COMPILE,
.new_schema = compile_schema_new,
@@ -251,16 +272,19 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map)
maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE);
maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP);
maat_kv_register(reserved_word_map, "flag", TABLE_TYPE_FLAG);
maat_kv_register(reserved_word_map, "flag_plus", TABLE_TYPE_FLAG_PLUS);
maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL);
maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS);
maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS);
maat_kv_register(reserved_word_map, "port", TABLE_TYPE_PORT);
maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN);
maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN);
maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL);
maat_kv_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION);
}
static void register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map,

2
test/maat_ex_data_gtest.cpp
View File

@@ -51,7 +51,7 @@ void ex_data_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
TEST(EXDataRuntime, Update) {
const char *table_name = "TEST_PLUGIN_EXDATA_TABLE";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ex_data_counter = 0;

121
test/maat_framework_gtest.cpp
View File

@@ -36,7 +36,7 @@ protected:
TEST_F(MaatFlagScan, basic) {
const char *flag_table_name = "FLAG_CONFIG";
int flag_table_id = maat_table_get_id(g_maat_instance, flag_table_name);
int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name);
//compile_id:192 flag: 0000 0001 mask: 0000 0011
//scan_data: 0000 1001 or 0000 1101 should hit
long long scan_data = 9;
@@ -78,8 +78,8 @@ TEST_F(MaatFlagScan, basic) {
TEST_F(MaatFlagScan, withExprRegion) {
const char *flag_table_name = "FLAG_CONFIG";
const char *expr_table_name = "HTTP_URL_LITERAL";
int flag_table_id = maat_table_get_id(g_maat_instance, flag_table_name);
int expr_table_id = maat_table_get_id(g_maat_instance, expr_table_name);
int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name);
int expr_table_id = maat_get_table_id(g_maat_instance, expr_table_name);
//compile_id:193 flag: 0000 0010 mask: 0000 0011
//scan_data: 0000 0010 or 0000 0100 should hit
long long flag_scan_data = 2;
@@ -109,7 +109,7 @@ TEST_F(MaatFlagScan, withExprRegion) {
TEST_F(MaatFlagScan, hitMultiCompile) {
const char *flag_table_name = "FLAG_CONFIG";
int flag_table_id = maat_table_get_id(g_maat_instance, flag_table_name);
int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name);
//compile_id:192 flag: 0000 0001 mask: 0000 0011
//compile_id:194 flag: 0001 0101 mask: 0001 1111
//scan_data: 0001 0101 should hit compile192 and compile194
@@ -140,7 +140,7 @@ TEST_F(MaatFlagScan, hitMultiCompile) {
TEST_F(MaatFlagScan, hitRepeatedCompile) {
const char *flag_table_name = "FLAG_CONFIG";
int flag_table_id = maat_table_get_id(g_maat_instance, flag_table_name);
int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
@@ -178,6 +178,41 @@ TEST_F(MaatFlagScan, hitRepeatedCompile) {
maat_state_free(&state);
}
TEST_F(MaatFlagScan, FlagPlus) {
const char *flag_table_name = "FLAG_PLUS_CONFIG";
const char *region_name = "I love China";
int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name);
//compile_id:196 flag: 0001 1111 mask: 0000 1111
//scan_data: 0000 1111 or 0001 1111 should hit
long long scan_data1 = 15;
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data1, results,
ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_ERR);
ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name));
ASSERT_EQ(ret, 0);
ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data1, results,
ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 196);
ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data1, results,
ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0};
int n_read = 0;
n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE);
EXPECT_NE(n_read, 0);
maat_state_free(&state);
}
class MaatStringScan : public testing::Test
{
protected:
@@ -191,7 +226,7 @@ protected:
TEST_F(MaatStringScan, Expr8) {
const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
@@ -216,7 +251,7 @@ TEST_F(MaatStringScan, Regex) {
struct maat_state *state = NULL;
const char *cookie = "Cookie: Txa123aheadBCAxd";
const char *table_name = "HTTP_URL_REGEX";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ret = maat_scan_string(g_maat_instance, table_id, 0, cookie, strlen(cookie),
results, ARRAY_SIZE, &n_hit_result, &state);
@@ -256,7 +291,7 @@ TEST_F(MaatStringScan, ExprPlus) {
const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=abckkk,1234567";
const char *scan_data2 = "Addis Sapphire Hotel";
const char *table_name = "HTTP_SIGNATURE";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, &state);
@@ -287,8 +322,8 @@ TEST_F(MaatStringScan, ExprAndExprPlus) {
const char *expr_plus_table_name = "HTTP_SIGNATURE";
const char *region_name = "I love China";
const char *scan_data = "today is Monday and yesterday is Tuesday";
int expr_table_id = maat_table_get_id(g_maat_instance, expr_table_name);
int expr_plus_table_id = maat_table_get_id(g_maat_instance, expr_plus_table_name);
int expr_table_id = maat_get_table_id(g_maat_instance, expr_table_name);
int expr_plus_table_id = maat_get_table_id(g_maat_instance, expr_plus_table_name);
int ret = maat_scan_string(g_maat_instance, expr_plus_table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
@@ -314,7 +349,7 @@ TEST_F(MaatStringScan, StreamInput) {
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
const char *table_name = "HTTP_URL_STREAM";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_scan_stream_open(g_maat_instance, table_id, 0);
@@ -354,7 +389,7 @@ TEST_F(MaatStringScan, ShouldNotHitExprPlus) {
0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a,
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a, 0xab, 0x00 };
int table_id = maat_table_get_id(g_maat_instance, "APP_PAYLOAD");
int table_id = maat_get_table_id(g_maat_instance, "APP_PAYLOAD");
ASSERT_GT(table_id, 0);
int ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name));
@@ -375,7 +410,7 @@ TEST_F(MaatStringScan, ExprPlusWithHex) {
const char *region_name1 = "Content-Type";
const char *region_name2 = "User-Agent";
int table_id = maat_table_get_id(g_maat_instance, "HTTP_SIGNATURE");
int table_id = maat_get_table_id(g_maat_instance, "HTTP_SIGNATURE");
ASSERT_GT(table_id, 0);
int ret = maat_state_set_scan_district(g_maat_instance, &state, region_name1, strlen(region_name1));
@@ -391,7 +426,7 @@ TEST_F(MaatStringScan, ExprPlusWithHex) {
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
table_id = maat_table_get_id(g_maat_instance, "KEYWORDS_TABLE");
table_id = maat_get_table_id(g_maat_instance, "KEYWORDS_TABLE");
ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data2, strlen(scan_data2),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
@@ -432,7 +467,7 @@ TEST_F(MaatStringScan, ExprPlusWithOffset)
0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a,
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a };
int table_id = maat_table_get_id(g_maat_instance, "APP_PAYLOAD");
int table_id = maat_get_table_id(g_maat_instance, "APP_PAYLOAD");
ASSERT_GT(table_id, 0);
int ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name));
@@ -452,7 +487,7 @@ TEST_F(MaatStringScan, ExprPlusWithOffset)
#endif
TEST_F(MaatStringScan, dynamic_config) {
const char *table_name = "HTTP_URL_LITERAL";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
char data[128] = "hello world";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
@@ -551,7 +586,7 @@ protected:
TEST_F(MaatIPScan, IPv4) {
const char *table_name = "IP_PLUS_CONFIG";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
char ip_str[32] = "10.0.7.100";
@@ -573,7 +608,7 @@ TEST_F(MaatIPScan, IPv4) {
TEST_F(MaatIPScan, IPv6) {
const char *table_name = "IP_PLUS_CONFIG";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
char ip_str[32] = "1001:da8:205:1::101";
uint8_t sip[16];
int ret = inet_pton(AF_INET6, ip_str, &sip);
@@ -593,7 +628,7 @@ TEST_F(MaatIPScan, IPv6) {
TEST_F(MaatIPScan, dynamic_config) {
const char *table_name = "IP_PLUS_CONFIG";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
char ip_str[32] = "100.100.100.100";
uint32_t sip;
int ret = inet_pton(AF_INET, ip_str, &sip);
@@ -693,7 +728,7 @@ TEST_F(MaatIntervalScan, Pure) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *table_name = "CONTENT_SIZE";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
unsigned int scan_data1 = 2015;
int ret = maat_scan_integer(g_maat_instance, table_id, 0, scan_data1, results, ARRAY_SIZE,
@@ -714,7 +749,7 @@ TEST_F(MaatIntervalScan, IntervalPlus) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *table_name = "INTERGER_PLUS";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
const char *district_str = "interval.plus";
int ret = maat_state_set_scan_district(g_maat_instance, &state, district_str, strlen(district_str));
@@ -750,8 +785,8 @@ TEST_F(NOTLogic, ScanNotAtLast) {
struct maat_state *state = NULL;
const char *hit_table_name = "HTTP_URL_LITERAL";
const char *not_hit_table_name = "KEYWORDS_TABLE";
int hit_table_id = maat_table_get_id(g_maat_instance, hit_table_name);
int not_hit_table_id = maat_table_get_id(g_maat_instance, not_hit_table_name);
int hit_table_id = maat_get_table_id(g_maat_instance, hit_table_name);
int not_hit_table_id = maat_get_table_id(g_maat_instance, not_hit_table_name);
int ret = maat_scan_string(g_maat_instance, hit_table_id, 0, string_should_hit, strlen(string_should_hit),
results, ARRAY_SIZE, &n_hit_result, &state);
@@ -818,7 +853,7 @@ protected:
TEST_F(PluginTable, Callback) {
const char *table_name = "QD_ENTRY_INFO";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
int ret = maat_table_callback_register(g_maat_instance, table_id,
maat_read_entry_start_cb,
@@ -887,7 +922,7 @@ void ip_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void *
TEST_F(IPPluginTable, EX_DATA) {
int ip_plugin_ex_data_counter = 0;
const char *table_name = "TEST_IP_PLUGIN_WITH_EXDATA";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id,
ip_plugin_EX_new_cb,
@@ -993,7 +1028,7 @@ void fqdn_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void
TEST_F(FQDNPluginTable, EX_DATA) {
const char *table_name = "TEST_FQDN_PLUGIN_WITH_EXDATA";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int fqdn_plugin_ex_data_counter = 0;
@@ -1086,7 +1121,7 @@ TEST_F(BoolPluginTable, EX_DATA) {
int ex_data_counter = 0, i = 0;
const char *table_name = "TEST_BOOL_PLUGIN_WITH_EXDATA";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id,
@@ -1146,7 +1181,7 @@ TEST_F(VirtualTable, basic) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *table_name = "HTTP_RESPONSE_KEYWORDS";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data),
@@ -1224,17 +1259,17 @@ void compile_ex_param_dup(int table_id, void **to, void **from, long argl, void
pthread_mutex_unlock(&(from_param->lock));
*((struct rule_ex_param**)to) = from_param;
}
#if 0
TEST_F(CompileTable, CompileEXData) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *table_name = "HTTP_URL_LITERAL";
const char *compile_table_name = "COMPILE";
const char *compile_table_name = "COMPILE_ALIAS";
const char *expect_name = "I have a name";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int compile_table_id = maat_table_get_id(g_maat_instance, compile_table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
int compile_table_id = maat_get_table_id(g_maat_instance, compile_table_name);
int ex_data_counter = 0;
int ret = maat_plugin_table_ex_schema_register(g_maat_instance, compile_table_id,
@@ -1243,6 +1278,10 @@ TEST_F(CompileTable, CompileEXData) {
compile_ex_param_dup,
0, &ex_data_counter);
ASSERT_TRUE(ret == 0);
EXPECT_EQ(ex_data_counter, 1);
ret = maat_state_set_scan_compile_tables(g_maat_instance, &state, &compile_table_name, 1);
EXPECT_EQ(ret, 0);
ret = maat_scan_string(g_maat_instance, table_id, 0, url, strlen(url),
results, ARRAY_SIZE, &n_hit_result, &state);
@@ -1262,7 +1301,7 @@ TEST_F(CompileTable, CompileEXData) {
maat_state_free(&state);
}
#endif
TEST_F(CompileTable, CompileRuleUpdate) {
//9999 0 0 0 0 0 anything 1 1 0.0
const char *compile_table_name = "COMPILE";
@@ -1328,7 +1367,7 @@ void accept_tags_entry_cb(int table_id, const char *table_line, void *u_para)
TEST_F(Policy, PluginRuleTags1) {
const char *table_name = "TEST_EFFECTIVE_RANGE_TABLE";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int callback_times=0;
@@ -1349,7 +1388,7 @@ void accept_tags_entry2_cb(int table_id, const char *table_line, void *u_para)
TEST_F(Policy, PluginRuleTags2) {
const char *table_name = "IR_INTERCEPT_IP";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int callback_times = 0;
@@ -1370,7 +1409,7 @@ TEST_F(Policy, CompileRuleTags) {
const char *should_not_hit = "string aaa should not hit";
const char *table_name = "HTTP_URL_LITERAL";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ret = maat_scan_string(g_maat_instance, table_id, 0, should_not_hit,
@@ -1394,11 +1433,11 @@ TEST_F(Policy, CompileEXData) {
const char *table_name = "HTTP_URL_LITERAL";
const char *expect_name = "I have a name";
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ex_data_counter = 0;
int compile_table_id = maat_table_get_id(g_maat_instance, "COMPILE");
int compile_table_id = maat_get_table_id(g_maat_instance, "COMPILE");
int ret = maat_plugin_table_ex_schema_register(g_maat_instance, compile_table_id,
compile_ex_param_new,
compile_ex_param_free,
@@ -1436,14 +1475,14 @@ TEST_F(Policy, SubGroup) {
uint32_t sip;
inet_pton(AF_INET,"10.0.6.205", &sip);
int table_id = maat_table_get_id(g_maat_instance, "MAIL_ADDR");
int table_id = maat_get_table_id(g_maat_instance, "MAIL_ADDR");
ASSERT_GT(table_id, 0);
int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
table_id = maat_table_get_id(g_maat_instance, "IP_CONFIG");
table_id = maat_get_table_id(g_maat_instance, "IP_CONFIG");
ASSERT_GT(table_id, 0);
ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE,
@@ -1511,7 +1550,7 @@ TEST_F(MaatCmdTest, PluginEXData) {
"2\t192.168.0.2\tliuqiangdong\t1\t0",
"3\t192.168.0.3\tmayun\t1\t0",
"4\t192.168.0.4\tliyanhong\t1\t0"};
int table_id = maat_table_get_id(g_maat_instance, table_name);
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
/* 1st line */

6
test/maat_input_mode_gtest.cpp
View File

@@ -37,7 +37,7 @@ TEST(json_mode, maat_scan_string) {
EXPECT_TRUE(maat_instance != NULL);
const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_table_get_id(maat_instance, table_name);
int table_id = maat_get_table_id(maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0};
size_t n_hit_result = 0;
@@ -82,7 +82,7 @@ TEST(iris_mode, maat_scan_string) {
EXPECT_TRUE(maat_instance != NULL);
const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_table_get_id(maat_instance, table_name);
int table_id = maat_get_table_id(maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0};
size_t n_hit_result = 0;
@@ -204,7 +204,7 @@ TEST(redis_mode, maat_scan_string) {
struct maat *maat_instance = maat_new(opts, table_info_path);
const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_table_get_id(maat_instance, table_name);
int table_id = maat_get_table_id(maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0};
size_t n_hit_result = 0;

75
test/maat_json.json
View File

@@ -322,31 +322,6 @@
}
]
},
{
"compile_id": 127,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "FILE_DIGEST",
"table_type": "digest",
"table_content": {
"raw_len": 1160164,
"digest": "3072:Xk/maCm4yLYtRIFDFnVfHH+CAQI6VD5mekDmaa/4qCuFnqak1s3/+Gn1IJHa/AvybUsbGWcIAy9grTp2s5bbj/TaKxONfb[0:1160163]#12288:UChtbFS6pypdTy4m2[0:1160163]",
"cfds_level": 70
}
}
]
}
]
},
{
"compile_id": 128,
"service": 1,
@@ -530,30 +505,6 @@
}
]
},
{
"compile_id": 135,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "SIM_URL",
"table_type": "similar",
"table_content": {
"target": "mwss.xiu.youku.com/live/hls/v1/0000000000000000000000001526a0a8/709.ts?&token=98765",
"threshold": 90
}
}
]
}
]
},
{
"compile_id": 136,
"service": 1,
@@ -693,10 +644,12 @@
"do_blacklist": 1,
"do_log": 1,
"user_region": "Something:I\\bhave\\ba\\bname,7799",
"compile_table_name": "COMPILE_ALIAS",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"g2c_table_name": "GROUP2COMPILE_ALIAS",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
@@ -2457,6 +2410,30 @@
]
}
]
},
{
"compile_id": 196,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "flag_plus",
"table_name": "FLAG_PLUS_CONFIG",
"table_content": {
"district": "I love\\bChina",
"flag": 30,
"flag_mask": 14
}
}
]
}
]
}
],
"plugin_table": [

432
test/table_info.conf
View File

@@ -27,170 +27,6 @@
},
{
"table_id":2,
"table_name":"GROUP2GROUP",
"table_type":"group2group",
"valid_column":3,
"custom": {
"group_id":1,
"super_group_id":2
}
},
{
"table_id":3,
"table_name":"HTTP_URL_LITERAL",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":4,
"table_name":"HTTP_URL_REGEX",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"regex",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":5,
"table_name":"HTTP_HOST",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":6,
"table_name":"KEYWORDS_TABLE",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":7,
"table_name":"IP_CONFIG",
"table_type":"ip_plus",
"valid_column":18,
"custom": {
"item_id":1,
"group_id":2,
"addr_type":3,
"saddr_format":4,
"sip1":5,
"sip2":6
}
},
{
"table_id":8,
"table_name":"CONTENT_SIZE",
"table_type":"intval",
"valid_column":5,
"custom": {
"item_id":1,
"group_id":2,
"low_bound":3,
"up_bound":4
}
},
{
"table_id":9,
"table_name":"QD_ENTRY_INFO",
"table_type":"plugin",
"valid_column":4,
"custom": {
"key":1,
"tag":3
}
},
{
"table_id":10,
"table_name":"HTTP_SIGNATURE",
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"district":3,
"keywords":4,
"expr_type":5,
"match_method":6,
"is_hexbin":7
}
},
{
"table_id":11,
"table_name":"IMAGE_FP",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":12,
"table_name":"TEST_EFFECTIVE_RANGE_TABLE",
"table_type":"plugin",
"valid_column":4,
"custom": {
"key":1,
"tag":5
}
},
{
"table_id":13,
"table_name":"TEST_FOREIGN_KEY",
"table_type":"plugin",
"valid_column":4,
"custom": {
"key":2,
"tag":3,
"foreign": [6,8]
}
},
{
"table_id":14,
"table_name":"COMPILE_ALIAS",
"table_type":"compile",
"user_region_encoded":"escape",
@@ -207,8 +43,186 @@
"evaluation_order":10
}
},
{
"table_id":3,
"table_name":"GROUP2COMPILE_ALIAS",
"table_type":"group2compile",
"associated_compile_table_id":0,
"valid_column":3,
"custom": {
"group_id":1,
"compile_id":2,
"not_flag":4,
"virtual_table_name":5,
"clause_index":6
}
},
{
"table_id":4,
"table_name":"GROUP2GROUP",
"table_type":"group2group",
"valid_column":3,
"custom": {
"group_id":1,
"super_group_id":2
}
},
{
"table_id":5,
"table_name":"HTTP_URL_LITERAL",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":6,
"table_name":"HTTP_URL_REGEX",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"regex",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":7,
"table_name":"HTTP_HOST",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":8,
"table_name":"KEYWORDS_TABLE",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":9,
"table_name":"IP_CONFIG",
"table_type":"ip_plus",
"valid_column":18,
"custom": {
"item_id":1,
"group_id":2,
"addr_type":3,
"saddr_format":4,
"sip1":5,
"sip2":6
}
},
{
"table_id":10,
"table_name":"CONTENT_SIZE",
"table_type":"intval",
"valid_column":5,
"custom": {
"item_id":1,
"group_id":2,
"low_bound":3,
"up_bound":4
}
},
{
"table_id":11,
"table_name":"QD_ENTRY_INFO",
"table_type":"plugin",
"valid_column":4,
"custom": {
"key":1,
"tag":3
}
},
{
"table_id":12,
"table_name":"HTTP_SIGNATURE",
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"district":3,
"keywords":4,
"expr_type":5,
"match_method":6,
"is_hexbin":7
}
},
{
"table_id":13,
"table_name":"IMAGE_FP",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":14,
"table_name":"TEST_EFFECTIVE_RANGE_TABLE",
"table_type":"plugin",
"valid_column":4,
"custom": {
"key":1,
"tag":5
}
},
{
"table_id":15,
"table_name":"TEST_FOREIGN_KEY",
"table_type":"plugin",
"valid_column":4,
"custom": {
"key":2,
"tag":3,
"foreign": [6,8]
}
},
{
"table_id":16,
"table_name":"TEST_PLUGIN_EXDATA_TABLE",
"table_type":"plugin",
"valid_column":4,
@@ -219,7 +233,7 @@
}
},
{
"table_id":16,
"table_id":17,
"table_name":"IR_INTERCEPT_IP",
"table_type":"plugin",
"valid_column":14,
@@ -229,7 +243,7 @@
}
},
{
"table_id":17,
"table_id":18,
"table_name":"APP_PAYLOAD",
"table_type":"expr_plus",
"valid_column":8,
@@ -246,7 +260,7 @@
}
},
{
"table_id":18,
"table_id":19,
"table_name":"TROJAN_PAYLOAD",
"table_type":"expr",
"valid_column":7,
@@ -263,7 +277,7 @@
}
},
{
"table_id":19,
"table_id":20,
"table_name":"MAIL_ADDR",
"table_type":"expr",
"valid_column":7,
@@ -279,7 +293,7 @@
}
},
{
"table_id":20,
"table_id":21,
"table_name":"IP_PLUS_CONFIG",
"table_type":"ip_plus",
"valid_column":18,
@@ -293,43 +307,43 @@
}
},
{
"table_id":21,
"table_id":22,
"table_name":"HTTP_RESPONSE_KEYWORDS",
"table_type":"virtual",
"physical_table": ["KEYWORDS_TABLE"]
},
{
"table_id":22,
"table_id":23,
"table_name":"HTTP_REQUEST_HEADER",
"table_type":"virtual",
"physical_table": ["HTTP_SIGNATURE"]
},
{
"table_id":23,
"table_id":24,
"table_name":"HTTP_RESPONSE_HEADER",
"table_type":"virtual",
"physical_table": ["HTTP_SIGNATURE"]
},
{
"table_id":24,
"table_id":25,
"table_name":"VIRTUAL_IP_PLUS_TABLE",
"table_type":"virtual",
"physical_table": ["IP_PLUS_CONFIG"]
},
{
"table_id":24,
"table_id":26,
"table_name":"VIRTUAL_IP_PLUS_SOURCE",
"table_type":"virtual",
"physical_table": ["IP_PLUS_CONFIG"]
},
{
"table_id":24,
"table_id":27,
"table_name":"VIRTUAL_IP_PLUS_DESTINATION",
"table_type":"virtual",
"physical_table": ["IP_PLUS_CONFIG"]
},
{
"table_id":29,
"table_id":28,
"table_name":"TEST_IP_PLUGIN_WITH_EXDATA",
"table_type":"ip_plugin",
"valid_column":6,
@@ -342,7 +356,7 @@
}
},
{
"table_id":30,
"table_id":29,
"table_name":"AS_NUMBER",
"table_type":"expr",
"valid_column":7,
@@ -358,19 +372,19 @@
}
},
{
"table_id":31,
"table_id":30,
"table_name":"SOURCE_IP_ASN",
"table_type":"virtual",
"physical_table":["AS_NUMBER"]
},
{
"table_id":32,
"table_id":31,
"table_name":"DESTINATION_IP_ASN",
"table_type":"virtual",
"physical_table":["AS_NUMBER"]
},
{
"table_id":33,
"table_id":32,
"table_name":"GeoLocation",
"table_type":"expr",
"valid_column":7,
@@ -386,13 +400,13 @@
}
},
{
"table_id":34,
"table_id":33,
"table_name":"SOURCE_IP_GEO",
"table_type":"virtual",
"physical_table":["GeoLocation"]
},
{
"table_id":35,
"table_id":34,
"table_name":"INTERGER_PLUS",
"table_type":"intval_plus",
"valid_column":6,
@@ -405,7 +419,7 @@
}
},
{
"table_id":36,
"table_id":35,
"table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA",
"table_type":"fqdn_plugin",
"valid_column":5,
@@ -416,7 +430,7 @@
}
},
{
"table_id":38,
"table_id":36,
"table_name":"APP_ID",
"table_type":"intval",
"valid_column":5,
@@ -428,7 +442,7 @@
}
},
{
"table_id":39,
"table_id":37,
"table_name":"EMPTY_KEYWORD",
"table_type":"expr",
"valid_column":7,
@@ -444,7 +458,7 @@
}
},
{
"table_id":40,
"table_id":38,
"table_name":"EMPTY_INTERGER",
"table_type":"intval",
"valid_column":5,
@@ -456,7 +470,7 @@
}
},
{
"table_id":41,
"table_id":39,
"table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA",
"table_type":"bool_plugin",
"valid_column":4,
@@ -466,7 +480,7 @@
}
},
{
"table_id":42,
"table_id":40,
"table_name":"FLAG_CONFIG",
"table_type":"flag",
"valid_column":5,
@@ -478,9 +492,22 @@
}
},
{
"table_id":43,
"table_name":"PORT_PROTO_CONFIG",
"table_type":"port_proto",
"table_id":41,
"table_name":"FLAG_PLUS_CONFIG",
"table_type":"flag_plus",
"valid_column":6,
"custom": {
"item_id":1,
"group_id":2,
"district":3,
"flag":4,
"flag_mask":5
}
},
{
"table_id":42,
"table_name":"PORT_CONFIG",
"table_type":"port",
"valid_column":7,
"custom": {
"item_id":1,
@@ -490,5 +517,28 @@
"port2":5,
"proto":6
}
},
{
"table_id":43,
"table_name":"VIRTUAL_PORT_SOURCE",
"table_type":"virtual",
"physical_table": ["PORT_CONFIG"]
},
{
"table_id":44,
"table_name":"VIRTUAL_PORT_DESTINATION",
"table_type":"virtual",
"physical_table": ["PORT_CONFIG"]
},
{
"table_id":45,
"table_name":"COMPOSITION_CONFIG",
"table_type":"composition",
"custom": {
"source_ip":"VIRTUAL_IP_PLUS_SOURCE",
"dest_ip":"VIRTUAL_IP_PLUS_DESTINATION",
"source_port":"VIRTUAL_PORT_SOURCE",
"dest_port":"VIRTUAL_PORT_DESTINATION"
}
}
]

2
tools/maat_redis_tool.cpp
View File

@@ -162,7 +162,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char
cur_table = rule_list[i].table_name;
}
fprintf(table_fp, "%s\tkey=%ld\n", rule_list[i].table_line, rule_list[i].rule_id);
fprintf(table_fp, "%s\tkey=%lld\n", rule_list[i].table_line, rule_list[i].rule_id);
line_count++;
}