gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add conjunction table

Browse Source
This commit is contained in:
liuwentan
2022-12-06 00:35:36 +08:00
parent 6d18cf0f36
commit 6ba2f6241e
5 changed files with 69 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/inc_internal/maat_table_schema.h
View File

@@ -166,6 +166,8 @@ struct table_item *table_schema_line_to_item(const char *line, struct table_sche
int table_schema_get_valid_flag_column(struct table_schema *table_schema); int table_schema_get_valid_flag_column(struct table_schema *table_schema);
void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name);
/* expr table schema API */ /* expr table schema API */
enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema); enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);

2
src/maat_rule.cpp
View File

@@ -115,6 +115,8 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param)
return -1; return -1;
} }
table_schema_set_updating_name(table_schema, table_name);
if (maat_instance->creating_maat_rt != NULL) { if (maat_instance->creating_maat_rt != NULL) {
maat_rt = maat_instance->creating_maat_rt; maat_rt = maat_instance->creating_maat_rt;
} else { } else {

87
src/maat_table_schema.cpp
View File

@@ -23,6 +23,7 @@
#define MAX_TABLE_LINE_SIZE (1024 * 16) #define MAX_TABLE_LINE_SIZE (1024 * 16)
#define MAX_FOREIGN_CLMN_NUM 8 #define MAX_FOREIGN_CLMN_NUM 8
#define MAX_CONJUNCTION_TABLE_NUM 8
struct expr_table_schema { struct expr_table_schema {
int item_id_column; int item_id_column;
@@ -66,7 +67,9 @@ struct virtual_table_schema {
struct table_schema { struct table_schema {
int table_id; int table_id;
char table_name[NAME_MAX]; int conj_cnt;
int updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][NAME_MAX];
enum table_type table_type; enum table_type table_type;
union { union {
struct expr_table_schema expr; struct expr_table_schema expr;
@@ -86,6 +89,7 @@ struct table_schema_manager {
struct table_schema *table_schema_new(void) struct table_schema *table_schema_new(void)
{ {
struct table_schema *ptable = ALLOC(struct table_schema, 1); struct table_schema *ptable = ALLOC(struct table_schema, 1);
ptable->conj_cnt = 1;
return ptable; return ptable;
} }
@@ -113,7 +117,7 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable,
fprintf(stderr, "table name %s length too long\n", json->valuestring); fprintf(stderr, "table name %s length too long\n", json->valuestring);
return -1; return -1;
} }
memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring)); memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring));
read_cnt++; read_cnt++;
} }
@@ -141,7 +145,7 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable,
json = cJSON_GetObjectItem(root, "rule"); json = cJSON_GetObjectItem(root, "rule");
if (json == NULL || json->type != cJSON_Object) { if (json == NULL || json->type != cJSON_Object) {
fprintf(stderr, "table %s has no rule\n", ptable->table_name); fprintf(stderr, "table %s has no rule\n", ptable->table_name[0]);
return -1; return -1;
} }
@@ -224,7 +228,7 @@ int read_plugin_table_schema(cJSON *root, struct table_schema *ptable)
fprintf(stderr, "table name %s length too long\n", json->valuestring); fprintf(stderr, "table name %s length too long\n", json->valuestring);
return -1; return -1;
} }
memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring)); memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring));
read_cnt++; read_cnt++;
} }
@@ -236,7 +240,7 @@ int read_plugin_table_schema(cJSON *root, struct table_schema *ptable)
json = cJSON_GetObjectItem(root, "rule"); json = cJSON_GetObjectItem(root, "rule");
if (json == NULL || json->type != cJSON_Object) { if (json == NULL || json->type != cJSON_Object) {
fprintf(stderr, "table %s has no rule\n", ptable->table_name); fprintf(stderr, "table %s has no rule\n", ptable->table_name[0]);
return -1; return -1;
} }
@@ -299,7 +303,7 @@ int read_ip_plugin_table_schema(cJSON *root, struct table_schema *ptable)
fprintf(stderr, "table name %s length too long\n", json->valuestring); fprintf(stderr, "table name %s length too long\n", json->valuestring);
return -1; return -1;
} }
memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring)); memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring));
read_cnt++; read_cnt++;
} }
@@ -311,7 +315,7 @@ int read_ip_plugin_table_schema(cJSON *root, struct table_schema *ptable)
json = cJSON_GetObjectItem(root, "rule"); json = cJSON_GetObjectItem(root, "rule");
if (NULL == json || json->type != cJSON_Object) { if (NULL == json || json->type != cJSON_Object) {
fprintf(stderr, "table %s has no rule\n", ptable->table_name); fprintf(stderr, "table %s has no rule\n", ptable->table_name[0]);
return -1; return -1;
} }
@@ -430,45 +434,55 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_
maat_kv_register(reserved_word_map, "stream", HS_SCAN_MODE_STREAM); maat_kv_register(reserved_word_map, "stream", HS_SCAN_MODE_STREAM);
struct table_schema_manager *table_schema_mgr = ALLOC(struct table_schema_manager, 1); struct table_schema_manager *table_schema_mgr = ALLOC(struct table_schema_manager, 1);
struct table_schema **pptable = table_schema_mgr->schema_table; struct table_schema **table_array = table_schema_mgr->schema_table;
table_schema_mgr->tablename2id_map = maat_kv_store_new(); table_schema_mgr->tablename2id_map = maat_kv_store_new();
struct table_schema *conj_table = NULL;
for (int i = 0; i < json_array_size; i++) { for (int i = 0; i < json_array_size; i++) {
cJSON *json = cJSON_GetArrayItem(root, i); cJSON *json = cJSON_GetArrayItem(root, i);
if (json != NULL && json->type == cJSON_Object) { if (json != NULL && json->type == cJSON_Object) {
struct table_schema *table_schema = table_schema_new(); struct table_schema *ptable = table_schema_new();
int ret = table_schema_populate(json, table_schema, reserved_word_map); int ret = table_schema_populate(json, ptable, reserved_word_map);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Maat populate table info error, table_id:%d\n", table_schema->table_id); fprintf(stderr, "Maat populate table info error, table_id:%d\n", ptable->table_id);
goto error; goto free_table;
} }
if (table_schema->table_id >= MAX_TABLE_NUM) { if (ptable->table_id >= MAX_TABLE_NUM) {
fprintf(stderr, "Maat read table info %s:%d error: table id %u > %d\n", fprintf(stderr, "Maat read table info %s:%d error: table id %u > %d\n",
table_info_path, i, table_schema->table_id, MAX_TABLE_NUM); table_info_path, i, ptable->table_id, MAX_TABLE_NUM);
goto error; goto free_table;
} }
ret = maat_kv_register(table_schema_mgr->tablename2id_map, table_schema->table_name, table_schema->table_id); ret = maat_kv_register(table_schema_mgr->tablename2id_map, ptable->table_name[0], ptable->table_id);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Duplicate table %s of table id %d\n", table_schema->table_name, table_schema->table_id); fprintf(stderr, "Duplicate table %s of table id %d\n", ptable->table_name[0], ptable->table_id);
goto error; goto free_table;
} }
/* TODO: conjunction table*/ /* TODO: conjunction table*/
#if 0 if (table_array[ptable->table_id] != NULL) {
if (pptable[table_schema->table_id] != NULL) { conj_table = table_array[ptable->table_id];
fprintf(stderr, "Duplicate table %s of table id %d", table_schema->table_name, table_schema->table_id); if (conj_table->conj_cnt == MAX_CONJUNCTION_TABLE_NUM) {
goto error; fprintf(stderr, "Maat read table info %s error: reach table_id %d conjunction upper limit\n",
table_info_path, ptable->table_id);
goto free_table;
}
memcpy(conj_table->table_name[conj_table->conj_cnt], ptable->table_name[0], NAME_MAX);
conj_table->conj_cnt++;
fprintf(stdout, "Maat read table info %s: conjunction %s with %s (id=%d,total=%d)\n",
table_info_path, ptable->table_name[0], conj_table->table_name[0],
conj_table->table_id, conj_table->conj_cnt);
goto free_table;
} }
#endif
pptable[table_schema->table_id] = table_schema; table_array[ptable->table_id] = ptable;
table_schema_mgr->n_schema_table++; table_schema_mgr->n_schema_table++;
continue; continue;
error: free_table:
table_schema_free(table_schema); table_schema_free(ptable);
ptable = NULL;
} }
} }
@@ -778,6 +792,21 @@ int table_schema_get_valid_flag_column(struct table_schema *table_schema)
return valid_flag_column; return valid_flag_column;
} }
void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name)
{
if (NULL == table_schema) {
return;
}
int i = 0;
for (i = 0; i < table_schema->conj_cnt; i++) {
if (0 == strcmp(table_schema->table_name[i], table_name)) {
table_schema->updating_name = i;
}
}
assert(i <= table_schema->conj_cnt);
}
int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema, struct expr_item *expr_item) int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema, struct expr_item *expr_item)
{ {
size_t column_offset = 0; size_t column_offset = 0;
@@ -1086,7 +1115,7 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS: case TABLE_TYPE_EXPR_PLUS:
case TABLE_TYPE_IP: case TABLE_TYPE_IP:
fprintf(stderr, "table %s is not plugin type\n", ptable->table_name); fprintf(stderr, "table_id: %d is not plugin type\n", table_id);
return -1; return -1;
default: default:
break; break;
@@ -1095,8 +1124,8 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m
struct plugin_table_schema *plugin_schema = &(ptable->plugin); struct plugin_table_schema *plugin_schema = &(ptable->plugin);
size_t idx = plugin_schema->cb_plug_cnt; size_t idx = plugin_schema->cb_plug_cnt;
if (idx == MAX_PLUGIN_PER_TABLE) { if (idx == MAX_PLUGIN_PER_TABLE) {
fprintf(stderr, "the plugin number of table %s exceed maxium:%d\n", fprintf(stderr, "the plugin number of table_id: %d exceed maxium:%d\n",
ptable->table_name, MAX_PLUGIN_PER_TABLE); ptable->table_id, MAX_PLUGIN_PER_TABLE);
return -1; return -1;
} }

35
test/maat_framework_gtest.cpp
View File

@@ -13,7 +13,7 @@ const char *rule_full_path = "./rule/full/index";
const char *rule_inc_path = "./rule/inc/index"; const char *rule_inc_path = "./rule/inc/index";
const char *json_path="./maat_json.json"; const char *json_path="./maat_json.json";
const char *iris_file = "./HTTP_URL.000001"; const char *iris_file = "./HTTP_URL.000001";
#if 0
TEST(maat_scan_string, hit_one_expr) { TEST(maat_scan_string, hit_one_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
@@ -49,7 +49,7 @@ TEST(maat_scan_string, hit_one_expr) {
EXPECT_EQ(n_result_array, 1); EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 103); */ EXPECT_EQ(result_array[0], 103); */
} }
#if 0
TEST(maat_scan_string, hit_two_expr) { TEST(maat_scan_string, hit_two_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
@@ -163,8 +163,7 @@ TEST(maat_scan_string, config_dynamic_update) {
EXPECT_EQ(result_array[3], 103); EXPECT_EQ(result_array[3], 103);
EXPECT_EQ(result_array[4], 101); EXPECT_EQ(result_array[4], 101);
} }
#endif
TEST(maat_scan_string, hit_one_expr) { TEST(maat_scan_string, hit_one_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
@@ -206,28 +205,6 @@ TEST(maat_scan_string, hit_three_expr) {
EXPECT_EQ(result_array[1], 27); EXPECT_EQ(result_array[1], 27);
EXPECT_EQ(result_array[2], 18); EXPECT_EQ(result_array[2], 18);
} }
#endif
TEST(maat_redis, write) {
char line[1024] = {0};
char table_name[64] = "HTTP_URL";
struct maat_cmd_line rule_line;
rule_line.table_name = (char *)malloc(sizeof(char) * 64);
memcpy((void *)rule_line.table_name, table_name, strlen(table_name));
FILE *fp = fopen(iris_file, "r");
EXPECT_NE(fp, nullptr);
int rule_num = 0;
fscanf(fp, "%d\n", &rule_num);
while (NULL != fgets(line, sizeof(line), fp)) {
rule_line.table_line = (char *)malloc(sizeof(line));
memcpy((void *)rule_line.table_line, line, strlen(line));
maat_cmd_set_line(g_maat_instance, &rule_line);
break;
}
}
int main(int argc, char ** argv) int main(int argc, char ** argv)
{ {
@@ -235,11 +212,7 @@ int main(int argc, char ** argv)
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
//maat_options_set_iris_full_dir(opts, rule_full_path); maat_options_set_json_file(opts, json_path);
//maat_options_set_iris_inc_dir(opts, rule_inc_path);
maat_options_set_redis_ip(opts, "127.0.0.1");
maat_options_set_redis_port(opts, 6379);
//maat_options_set_json_file(opts, json_path);
g_maat_instance = maat_new(opts, table_info_path); g_maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(g_maat_instance, nullptr); EXPECT_NE(g_maat_instance, nullptr);

3
test/maat_input_mode_gtest.cpp
View File

@@ -144,6 +144,9 @@ TEST(redis_mode, maat_scan_string) {
redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db); redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db);
EXPECT_NE(c, nullptr); EXPECT_NE(c, nullptr);
redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb");
EXPECT_NE(reply, nullptr);
if ((access(json_iris_path, F_OK)) < 0) { if ((access(json_iris_path, F_OK)) < 0) {
char tmp_iris_path[128] = {0}; char tmp_iris_path[128] = {0};
char *json_buff = NULL; char *json_buff = NULL;