gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix foreign key rewrite fail

Browse Source
This commit is contained in:
root
2024-10-16 01:48:31 +00:00
parent 78f733417c
commit cc67447c4e
6 changed files with 105 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
scanner/expr_matcher/adapter_rs/adapter_rs.cpp
View File

@@ -64,7 +64,7 @@ struct rs_lit_engine {
size_t n_thread; size_t n_thread;
rs_database_t *rs_db; rs_database_t *rs_db;
struct bloom **blooms; struct bloom **blooms;
struct rs_lit_stream **streams; /* per thread */ struct rs_lit_stream **per_thread_scratch_streams; /* per thread */
struct pattern_attribute *ref_pat_attr; struct pattern_attribute *ref_pat_attr;
struct log_handle *logger; struct log_handle *logger;
}; };
@@ -309,14 +309,14 @@ void rs_lit_engine_free(void *rs_lit_engine)
FREE(rs_lit_inst->blooms); FREE(rs_lit_inst->blooms);
} }
if (rs_lit_inst->streams != NULL) { if (rs_lit_inst->per_thread_scratch_streams != NULL) {
for (size_t i = 0; i < rs_lit_inst->n_thread; i++) { for (size_t i = 0; i < rs_lit_inst->n_thread; i++) {
if (rs_lit_inst->streams[i] != NULL) { if (rs_lit_inst->per_thread_scratch_streams[i] != NULL) {
rs_lit_stream_close(rs_lit_inst->streams[i]); rs_lit_stream_close(rs_lit_inst->per_thread_scratch_streams[i]);
rs_lit_inst->streams[i] = NULL; rs_lit_inst->per_thread_scratch_streams[i] = NULL;
} }
} }
FREE(rs_lit_inst->streams); FREE(rs_lit_inst->per_thread_scratch_streams);
} }
FREE(rs_lit_inst); FREE(rs_lit_inst);
@@ -341,9 +341,9 @@ void *rs_lit_engine_new(struct expr_rule *rules, size_t n_rule,
bloom_init2(rs_lit_inst->blooms[i], 1024, 0.001); bloom_init2(rs_lit_inst->blooms[i], 1024, 0.001);
} }
rs_lit_inst->streams = ALLOC(struct rs_lit_stream *, n_thread); rs_lit_inst->per_thread_scratch_streams = ALLOC(struct rs_lit_stream *, n_thread);
for (size_t i = 0; i < n_thread; i++) { for (size_t i = 0; i < n_thread; i++) {
rs_lit_inst->streams[i] = (struct rs_lit_stream *)rs_lit_stream_open(rs_lit_inst, i); rs_lit_inst->per_thread_scratch_streams[i] = (struct rs_lit_stream *)rs_lit_stream_open(rs_lit_inst, i);
} }
return rs_lit_inst; return rs_lit_inst;
@@ -360,7 +360,7 @@ int rs_lit_engine_scan(void *rs_lit_engine, int thread_id,
} }
struct rs_lit_engine *rs_lit_inst = (struct rs_lit_engine *)rs_lit_engine; struct rs_lit_engine *rs_lit_inst = (struct rs_lit_engine *)rs_lit_engine;
struct rs_lit_stream *rs_lit_stream = rs_lit_inst->streams[thread_id]; struct rs_lit_stream *rs_lit_stream = rs_lit_inst->per_thread_scratch_streams[thread_id];
assert(rs_lit_stream != NULL); assert(rs_lit_stream != NULL);
utarray_clear(rs_lit_stream->matched_pat->pattern_ids); utarray_clear(rs_lit_stream->matched_pat->pattern_ids);

1
src/inc_internal/maat_redis_monitor.h
View File

@@ -26,6 +26,7 @@ extern "C"
#include "uthash/uthash.h" #include "uthash/uthash.h"
struct foreign_key { struct foreign_key {
char *foreign_name;
char *key; char *key;
size_t key_len; size_t key_len;
char *filename; char *filename;

7
src/maat_plugin.c
View File

@@ -222,9 +222,10 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema)
} }
} }
int plugin_table_get_foreign_names(struct plugin_schema *plugin_schema, int plugin_table_get_foreign_names(struct plugin_schema *plugin_schema, char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1])
char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1])
{ {
int n_foreign_keys = 0;
if (NULL == plugin_schema) { if (NULL == plugin_schema) {
return -1; return -1;
} }
@@ -233,7 +234,7 @@ int plugin_table_get_foreign_names(struct plugin_schema *plugin_schema,
strncpy(foreign_names[i], plugin_schema->foreign_names[i], MAX_FOREIGN_NAME_LEN); strncpy(foreign_names[i], plugin_schema->foreign_names[i], MAX_FOREIGN_NAME_LEN);
} }
return plugin_schema->n_foreign; return n_foreign_keys;
} }
int plugin_table_set_ex_container_schema(void *plugin_schema, int table_id, int plugin_table_set_ex_container_schema(void *plugin_schema, int table_id,

63
src/maat_redis_monitor.c
View File

@@ -64,43 +64,65 @@ get_foreign_cont_filename(const char *table_name, const char* rule_uuid_str,
} }
static void static void
get_foregin_keys(struct serial_rule *p_rule, const char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1], get_foregin_keys(struct serial_rule *p_rule, char foreign_names[][MAX_FOREIGN_NAME_LEN + 1],
int n_foreign, const char *dir, struct log_handle *logger) int n_foreign, const char *dir, struct log_handle *logger)
{ {
int foreign_key_size = 0; int foreign_key_size = 0;
p_rule->f_keys = ALLOC(struct foreign_key, n_foreign); p_rule->f_keys = ALLOC(struct foreign_key, n_foreign);
for (int i = 0; i < n_foreign; i++) { for (int i = 0; i < n_foreign; i++) {
const char *p_foreign = foreign_names[i]; const char *p_foreign_name = foreign_names[i];
cJSON *json = cJSON_Parse(p_rule->table_line);
foreign_key_size = strlen(p_foreign); if (json == NULL) {
//emtpy file log_fatal(logger, MODULE_REDIS_MONITOR,
if (0 == strncasecmp(p_foreign, "null", strlen("null"))) { "[%s:%d] Get %s,%s foreign key failed: "
"Invalid table line", __FUNCTION__, __LINE__,
p_rule->table_name, p_rule->rule_uuid_str);
continue; continue;
} }
if (0 != strncmp(p_foreign, foreign_source_prefix, cJSON *item = cJSON_GetObjectItem(json, p_foreign_name);
if (item == NULL || item->type != cJSON_String) {
log_fatal(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Get %s,%s foreign key failed: "
"Invalid foreign key %s", __FUNCTION__, __LINE__,
p_rule->table_name, p_rule->rule_uuid_str, p_foreign_name);
continue;
}
const char *p_foreign_key = item->valuestring;
foreign_key_size = strlen(p_foreign_key);
//emtpy file
if (0 == strncasecmp(p_foreign_key, "null", strlen("null"))) {
continue;
}
if (0 != strncmp(p_foreign_key, foreign_source_prefix,
strlen(foreign_source_prefix))) { strlen(foreign_source_prefix))) {
log_fatal(logger, MODULE_REDIS_MONITOR, log_fatal(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Get %s,%s foreign key failed: " "[%s:%d] Get %s,%s foreign key failed: "
"Invalid source prefix %s", __FUNCTION__, __LINE__, "Invalid source prefix %s", __FUNCTION__, __LINE__,
p_rule->table_name, p_rule->rule_uuid_str, p_foreign); p_rule->table_name, p_rule->rule_uuid_str, p_foreign_key);
continue; continue;
} }
foreign_key_size = foreign_key_size - strlen(foreign_source_prefix); foreign_key_size = foreign_key_size - strlen(foreign_source_prefix);
p_foreign += strlen(foreign_source_prefix); p_foreign_key += strlen(foreign_source_prefix);
if (0 != strncmp(p_foreign, foreign_key_prefix, if (0 != strncmp(p_foreign_key, foreign_key_prefix,
strlen(foreign_key_prefix))) { strlen(foreign_key_prefix))) {
log_info(logger, MODULE_REDIS_MONITOR, log_info(logger, MODULE_REDIS_MONITOR,
"[%s:%d] %s, %s foreign key prefix %s is not recommended", "[%s:%d] %s, %s foreign key prefix %s is not recommended",
__FUNCTION__, __LINE__, p_rule->table_name, p_rule->rule_uuid_str, __FUNCTION__, __LINE__, p_rule->table_name, p_rule->rule_uuid_str,
p_foreign); p_foreign_key);
} }
p_rule->f_keys[p_rule->n_foreign].foreign_name = ALLOC(char, strlen(p_foreign_name) + 1);
memcpy(p_rule->f_keys[p_rule->n_foreign].foreign_name, p_foreign_name, strlen(p_foreign_name));
p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size + 1); p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size + 1);
memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign, foreign_key_size); memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign_key, foreign_key_size);
p_rule->f_keys[p_rule->n_foreign].filename = p_rule->f_keys[p_rule->n_foreign].filename =
get_foreign_cont_filename(p_rule->table_name, p_rule->rule_uuid_str, get_foreign_cont_filename(p_rule->table_name, p_rule->rule_uuid_str,
p_rule->f_keys[p_rule->n_foreign].key, dir); p_rule->f_keys[p_rule->n_foreign].key, dir);
@@ -117,6 +139,7 @@ get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, struct maat *maat_inst, const char *dir) int rule_num, struct maat *maat_inst, const char *dir)
{ {
int rule_with_foreign_key = 0; int rule_with_foreign_key = 0;
char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1] = {0};
for (int i = 0; i < rule_num; i++) { for (int i = 0; i < rule_num; i++) {
if (NULL == rule_list[i].table_line) { if (NULL == rule_list[i].table_line) {
@@ -134,10 +157,8 @@ get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list,
continue; continue;
} }
char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1];
int n_foreign = int n_foreign =
plugin_table_get_foreign_names(schema, plugin_table_get_foreign_names(schema, foreign_names);
foreign_names);
if (0 == n_foreign) { if (0 == n_foreign) {
continue; continue;
} }
@@ -158,7 +179,7 @@ int maat_get_foreign_keys_by_prefix(redisContext *ctx,
int rule_with_foreign_key = 0; int rule_with_foreign_key = 0;
int n_foreign = 0; int n_foreign = 0;
char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1]; char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1] = {0};
for (int i = 0; i < rule_num; i++) { for (int i = 0; i < rule_num; i++) {
n_foreign = 0; n_foreign = 0;
@@ -170,10 +191,9 @@ int maat_get_foreign_keys_by_prefix(redisContext *ctx,
cJSON_ArrayForEach(item, json) { cJSON_ArrayForEach(item, json) {
if (item->type == cJSON_String) { if (item->type == cJSON_String) {
if (strlen(item->valuestring) > strlen(foreign_source_prefix) && if (strlen(item->valuestring) > strlen(foreign_source_prefix) &&
0 == strncmp(item->valuestring, foreign_key_prefix, strlen(foreign_key_prefix))) { 0 == strncmp(item->valuestring, foreign_source_prefix, strlen(foreign_source_prefix))) {
strncpy(foreign_names[n_foreign], item->valuestring, strncpy(foreign_names[n_foreign], item->string, strlen(item->string));
MAX_FOREIGN_NAME_LEN);
n_foreign++; n_foreign++;
} }
} }
@@ -436,6 +456,7 @@ void maat_clear_rule_cache(struct serial_rule *s_rule)
for (int i = 0; i < s_rule->n_foreign; i++) { for (int i = 0; i < s_rule->n_foreign; i++) {
FREE(s_rule->f_keys[i].filename); FREE(s_rule->f_keys[i].filename);
FREE(s_rule->f_keys[i].key); FREE(s_rule->f_keys[i].key);
FREE(s_rule->f_keys[i].foreign_name);
} }
FREE(s_rule->f_keys); FREE(s_rule->f_keys);
@@ -762,13 +783,13 @@ void maat_rewrite_table_line_with_foreign(struct serial_rule *s_rule)
} }
for (i = 0; i < s_rule->n_foreign; i++) { for (i = 0; i < s_rule->n_foreign; i++) {
cJSON *tmp_obj = cJSON_GetObjectItem(json, s_rule->f_keys[i].key); cJSON *tmp_obj = cJSON_GetObjectItem(json, s_rule->f_keys[i].foreign_name);
if (NULL == tmp_obj) { if (NULL == tmp_obj) {
continue; continue;
} }
cJSON_DeleteItemFromObject(json, s_rule->f_keys[i].key); cJSON_DeleteItemFromObject(json, s_rule->f_keys[i].foreign_name);
cJSON_AddStringToObject(json, s_rule->f_keys[i].key, s_rule->f_keys[i].filename); cJSON_AddStringToObject(json, s_rule->f_keys[i].foreign_name, s_rule->f_keys[i].filename);
} }
char *rewrite_line_json = cJSON_PrintUnformatted(json); char *rewrite_line_json = cJSON_PrintUnformatted(json);

49
test/maat_framework_gtest.cpp
View File

@@ -487,6 +487,7 @@ protected:
maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_hit_path_enabled(opts); maat_options_set_hit_path_enabled(opts);
maat_options_set_hit_object_enabled(opts);
maat_options_set_expr_engine(opts, MAAT_EXPR_ENGINE_HS); maat_options_set_expr_engine(opts, MAAT_EXPR_ENGINE_HS);
_shared_maat_inst = maat_new(opts, g_table_info_path); _shared_maat_inst = maat_new(opts, g_table_info_path);
@@ -1189,6 +1190,54 @@ TEST_F(HsStringScan, StreamInput) {
state = NULL; state = NULL;
} }
TEST_F(HsStringScan, StreamHitDirectObject) {
uuid_t results[ARRAY_SIZE];
size_t n_hit_result = 0;
int thread_id = 0;
int ret;
struct maat *maat_inst = HsStringScan::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
const char *table_name = "HTTP_URL";
const char *attribute_name = "HTTP_URL";
const char *scan_data1 = "www.3300av.com";
const char *scan_data2 = "sdadhuadhasdgufgh;sdfhjaufhiwebfiusdafhaos;dhfaluhjweh";
memset(results, 0, sizeof(results));
struct maat_stream *sp = maat_stream_new(maat_inst, table_name, attribute_name, state);
ASSERT_TRUE(sp != NULL);
ret = maat_stream_scan(sp, scan_data1, strlen(scan_data1), results,
ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
char uuid_str[UUID_STR_LEN] = {0};
uuid_unparse(results[0], uuid_str);
EXPECT_STREQ(uuid_str, "00000000-0000-0000-0000-000000000133");
struct maat_hit_object object_array[ARRAY_SIZE];
ret = maat_state_get_direct_hit_objects(state, object_array, ARRAY_SIZE);
EXPECT_EQ(ret, 1);
uuid_unparse(object_array[0].object_uuid, uuid_str);
EXPECT_STREQ(uuid_str, "00000000-0000-0000-0000-000000000112");
ret = maat_scan_not_logic(maat_inst, table_name, attribute_name, results, ARRAY_SIZE,
&n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
ret = maat_stream_scan(sp, scan_data2, strlen(scan_data2), results,
ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
ret = maat_state_get_direct_hit_objects(state, object_array, ARRAY_SIZE);
EXPECT_EQ(ret, 0);
maat_stream_free(sp);
maat_state_free(state);
state = NULL;
}
#if 0 //TODO #if 0 //TODO
TEST_F(HsStringScan, dynamic_config) { TEST_F(HsStringScan, dynamic_config) {
const char *table_name = "HTTP_URL"; const char *table_name = "HTTP_URL";

1
test/table_info.json
View File

@@ -62,7 +62,6 @@
"table_name":"HTTP_REGION", "table_name":"HTTP_REGION",
"db_tables":["HTTP_URL", "HTTP_HOST"], "db_tables":["HTTP_URL", "HTTP_HOST"],
"table_type":"expr", "table_type":"expr",
"expr_engine":"rulescan",
"schema_tag": "{\"http_region\": \"expr\"}" "schema_tag": "{\"http_region\": \"expr\"}"
}, },
{ {