gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add conjunction table

Browse Source
This commit is contained in:
liuwentan
2022-12-06 00:35:36 +08:00
parent 6d18cf0f36
commit 6ba2f6241e
5 changed files with 69 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/inc_internal/maat_table_schema.h
View File

@@ -166,6 +166,8 @@ struct table_item *table_schema_line_to_item(const char *line, struct table_sche
int table_schema_get_valid_flag_column(struct table_schema *table_schema);
void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name);
/* expr table schema API */
enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);

2
src/maat_rule.cpp
View File

@@ -115,6 +115,8 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param)
return -1;
}
table_schema_set_updating_name(table_schema, table_name);
if (maat_instance->creating_maat_rt != NULL) {
maat_rt = maat_instance->creating_maat_rt;
} else {

87
src/maat_table_schema.cpp
View File

@@ -23,6 +23,7 @@
#define MAX_TABLE_LINE_SIZE (1024 * 16)
#define MAX_FOREIGN_CLMN_NUM 8
#define MAX_CONJUNCTION_TABLE_NUM 8
struct expr_table_schema {
int item_id_column;
@@ -66,7 +67,9 @@ struct virtual_table_schema {
struct table_schema {
int table_id;
char table_name[NAME_MAX];
int conj_cnt;
int updating_name;
char table_name[MAX_CONJUNCTION_TABLE_NUM][NAME_MAX];
enum table_type table_type;
union {
struct expr_table_schema expr;
@@ -86,6 +89,7 @@ struct table_schema_manager {
struct table_schema *table_schema_new(void)
{
struct table_schema *ptable = ALLOC(struct table_schema, 1);
ptable->conj_cnt = 1;
return ptable;
}
@@ -113,7 +117,7 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable,
fprintf(stderr, "table name %s length too long\n", json->valuestring);
return -1;
}
memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring));
memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring));
read_cnt++;
}
@@ -141,7 +145,7 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable,
json = cJSON_GetObjectItem(root, "rule");
if (json == NULL || json->type != cJSON_Object) {
fprintf(stderr, "table %s has no rule\n", ptable->table_name);
fprintf(stderr, "table %s has no rule\n", ptable->table_name[0]);
return -1;
}
@@ -224,7 +228,7 @@ int read_plugin_table_schema(cJSON *root, struct table_schema *ptable)
fprintf(stderr, "table name %s length too long\n", json->valuestring);
return -1;
}
memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring));
memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring));
read_cnt++;
}
@@ -236,7 +240,7 @@ int read_plugin_table_schema(cJSON *root, struct table_schema *ptable)
json = cJSON_GetObjectItem(root, "rule");
if (json == NULL || json->type != cJSON_Object) {
fprintf(stderr, "table %s has no rule\n", ptable->table_name);
fprintf(stderr, "table %s has no rule\n", ptable->table_name[0]);
return -1;
}
@@ -299,7 +303,7 @@ int read_ip_plugin_table_schema(cJSON *root, struct table_schema *ptable)
fprintf(stderr, "table name %s length too long\n", json->valuestring);
return -1;
}
memcpy(ptable->table_name, json->valuestring, strlen(json->valuestring));
memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring));
read_cnt++;
}
@@ -311,7 +315,7 @@ int read_ip_plugin_table_schema(cJSON *root, struct table_schema *ptable)
json = cJSON_GetObjectItem(root, "rule");
if (NULL == json || json->type != cJSON_Object) {
fprintf(stderr, "table %s has no rule\n", ptable->table_name);
fprintf(stderr, "table %s has no rule\n", ptable->table_name[0]);
return -1;
}
@@ -430,45 +434,55 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_
maat_kv_register(reserved_word_map, "stream", HS_SCAN_MODE_STREAM);
struct table_schema_manager *table_schema_mgr = ALLOC(struct table_schema_manager, 1);
struct table_schema **pptable = table_schema_mgr->schema_table;
struct table_schema **table_array = table_schema_mgr->schema_table;
table_schema_mgr->tablename2id_map = maat_kv_store_new();
struct table_schema *conj_table = NULL;
for (int i = 0; i < json_array_size; i++) {
cJSON *json = cJSON_GetArrayItem(root, i);
if (json != NULL && json->type == cJSON_Object) {
struct table_schema *table_schema = table_schema_new();
int ret = table_schema_populate(json, table_schema, reserved_word_map);
struct table_schema *ptable = table_schema_new();
int ret = table_schema_populate(json, ptable, reserved_word_map);
if (ret < 0) {
fprintf(stderr, "Maat populate table info error, table_id:%d\n", table_schema->table_id);
goto error;
fprintf(stderr, "Maat populate table info error, table_id:%d\n", ptable->table_id);
goto free_table;
}
if (table_schema->table_id >= MAX_TABLE_NUM) {
if (ptable->table_id >= MAX_TABLE_NUM) {
fprintf(stderr, "Maat read table info %s:%d error: table id %u > %d\n",
table_info_path, i, table_schema->table_id, MAX_TABLE_NUM);
goto error;
table_info_path, i, ptable->table_id, MAX_TABLE_NUM);
goto free_table;
}
ret = maat_kv_register(table_schema_mgr->tablename2id_map, table_schema->table_name, table_schema->table_id);
ret = maat_kv_register(table_schema_mgr->tablename2id_map, ptable->table_name[0], ptable->table_id);
if (ret < 0) {
fprintf(stderr, "Duplicate table %s of table id %d\n", table_schema->table_name, table_schema->table_id);
goto error;
fprintf(stderr, "Duplicate table %s of table id %d\n", ptable->table_name[0], ptable->table_id);
goto free_table;
}
/* TODO: conjunction table*/
#if 0
if (pptable[table_schema->table_id] != NULL) {
fprintf(stderr, "Duplicate table %s of table id %d", table_schema->table_name, table_schema->table_id);
goto error;
if (table_array[ptable->table_id] != NULL) {
conj_table = table_array[ptable->table_id];
if (conj_table->conj_cnt == MAX_CONJUNCTION_TABLE_NUM) {
fprintf(stderr, "Maat read table info %s error: reach table_id %d conjunction upper limit\n",
table_info_path, ptable->table_id);
goto free_table;
}
memcpy(conj_table->table_name[conj_table->conj_cnt], ptable->table_name[0], NAME_MAX);
conj_table->conj_cnt++;
fprintf(stdout, "Maat read table info %s: conjunction %s with %s (id=%d,total=%d)\n",
table_info_path, ptable->table_name[0], conj_table->table_name[0],
conj_table->table_id, conj_table->conj_cnt);
goto free_table;
}
#endif
pptable[table_schema->table_id] = table_schema;
table_array[ptable->table_id] = ptable;
table_schema_mgr->n_schema_table++;
continue;
error:
table_schema_free(table_schema);
free_table:
table_schema_free(ptable);
ptable = NULL;
}
}
@@ -778,6 +792,21 @@ int table_schema_get_valid_flag_column(struct table_schema *table_schema)
return valid_flag_column;
}
void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name)
{
if (NULL == table_schema) {
return;
}
int i = 0;
for (i = 0; i < table_schema->conj_cnt; i++) {
if (0 == strcmp(table_schema->table_name[i], table_name)) {
table_schema->updating_name = i;
}
}
assert(i <= table_schema->conj_cnt);
}
int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema, struct expr_item *expr_item)
{
size_t column_offset = 0;
@@ -1086,7 +1115,7 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
case TABLE_TYPE_IP:
fprintf(stderr, "table %s is not plugin type\n", ptable->table_name);
fprintf(stderr, "table_id: %d is not plugin type\n", table_id);
return -1;
default:
break;
@@ -1095,8 +1124,8 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m
struct plugin_table_schema *plugin_schema = &(ptable->plugin);
size_t idx = plugin_schema->cb_plug_cnt;
if (idx == MAX_PLUGIN_PER_TABLE) {
fprintf(stderr, "the plugin number of table %s exceed maxium:%d\n",
ptable->table_name, MAX_PLUGIN_PER_TABLE);
fprintf(stderr, "the plugin number of table_id: %d exceed maxium:%d\n",
ptable->table_id, MAX_PLUGIN_PER_TABLE);
return -1;
}

35
test/maat_framework_gtest.cpp
View File

@@ -13,7 +13,7 @@ const char *rule_full_path = "./rule/full/index";
const char *rule_inc_path = "./rule/inc/index";
const char *json_path="./maat_json.json";
const char *iris_file = "./HTTP_URL.000001";
#if 0
TEST(maat_scan_string, hit_one_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
@@ -49,7 +49,7 @@ TEST(maat_scan_string, hit_one_expr) {
EXPECT_EQ(n_result_array, 1);
EXPECT_EQ(result_array[0], 103); */
}
#if 0
TEST(maat_scan_string, hit_two_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
@@ -163,8 +163,7 @@ TEST(maat_scan_string, config_dynamic_update) {
EXPECT_EQ(result_array[3], 103);
EXPECT_EQ(result_array[4], 101);
}
#endif
TEST(maat_scan_string, hit_one_expr) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
@@ -206,28 +205,6 @@ TEST(maat_scan_string, hit_three_expr) {
EXPECT_EQ(result_array[1], 27);
EXPECT_EQ(result_array[2], 18);
}
#endif
TEST(maat_redis, write) {
char line[1024] = {0};
char table_name[64] = "HTTP_URL";
struct maat_cmd_line rule_line;
rule_line.table_name = (char *)malloc(sizeof(char) * 64);
memcpy((void *)rule_line.table_name, table_name, strlen(table_name));
FILE *fp = fopen(iris_file, "r");
EXPECT_NE(fp, nullptr);
int rule_num = 0;
fscanf(fp, "%d\n", &rule_num);
while (NULL != fgets(line, sizeof(line), fp)) {
rule_line.table_line = (char *)malloc(sizeof(line));
memcpy((void *)rule_line.table_line, line, strlen(line));
maat_cmd_set_line(g_maat_instance, &rule_line);
break;
}
}
int main(int argc, char ** argv)
{
@@ -235,11 +212,7 @@ int main(int argc, char ** argv)
::testing::InitGoogleTest(&argc, argv);
struct maat_options *opts = maat_options_new();
//maat_options_set_iris_full_dir(opts, rule_full_path);
//maat_options_set_iris_inc_dir(opts, rule_inc_path);
maat_options_set_redis_ip(opts, "127.0.0.1");
maat_options_set_redis_port(opts, 6379);
//maat_options_set_json_file(opts, json_path);
maat_options_set_json_file(opts, json_path);
g_maat_instance = maat_new(opts, table_info_path);
EXPECT_NE(g_maat_instance, nullptr);

3
test/maat_input_mode_gtest.cpp
View File

@@ -144,6 +144,9 @@ TEST(redis_mode, maat_scan_string) {
redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db);
EXPECT_NE(c, nullptr);
redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb");
EXPECT_NE(reply, nullptr);
if ((access(json_iris_path, F_OK)) < 0) {
char tmp_iris_path[128] = {0};
char *json_buff = NULL;