diff --git a/scanner/expr_matcher/adapter_rs/adapter_rs.cpp b/scanner/expr_matcher/adapter_rs/adapter_rs.cpp index 8b49f8e..76e0c80 100644 --- a/scanner/expr_matcher/adapter_rs/adapter_rs.cpp +++ b/scanner/expr_matcher/adapter_rs/adapter_rs.cpp @@ -64,7 +64,7 @@ struct rs_lit_engine { size_t n_thread; rs_database_t *rs_db; struct bloom **blooms; - struct rs_lit_stream **streams; /* per thread */ + struct rs_lit_stream **per_thread_scratch_streams; /* per thread */ struct pattern_attribute *ref_pat_attr; struct log_handle *logger; }; @@ -309,14 +309,14 @@ void rs_lit_engine_free(void *rs_lit_engine) FREE(rs_lit_inst->blooms); } - if (rs_lit_inst->streams != NULL) { + if (rs_lit_inst->per_thread_scratch_streams != NULL) { for (size_t i = 0; i < rs_lit_inst->n_thread; i++) { - if (rs_lit_inst->streams[i] != NULL) { - rs_lit_stream_close(rs_lit_inst->streams[i]); - rs_lit_inst->streams[i] = NULL; + if (rs_lit_inst->per_thread_scratch_streams[i] != NULL) { + rs_lit_stream_close(rs_lit_inst->per_thread_scratch_streams[i]); + rs_lit_inst->per_thread_scratch_streams[i] = NULL; } } - FREE(rs_lit_inst->streams); + FREE(rs_lit_inst->per_thread_scratch_streams); } FREE(rs_lit_inst); @@ -341,9 +341,9 @@ void *rs_lit_engine_new(struct expr_rule *rules, size_t n_rule, bloom_init2(rs_lit_inst->blooms[i], 1024, 0.001); } - rs_lit_inst->streams = ALLOC(struct rs_lit_stream *, n_thread); + rs_lit_inst->per_thread_scratch_streams = ALLOC(struct rs_lit_stream *, n_thread); for (size_t i = 0; i < n_thread; i++) { - rs_lit_inst->streams[i] = (struct rs_lit_stream *)rs_lit_stream_open(rs_lit_inst, i); + rs_lit_inst->per_thread_scratch_streams[i] = (struct rs_lit_stream *)rs_lit_stream_open(rs_lit_inst, i); } return rs_lit_inst; @@ -360,7 +360,7 @@ int rs_lit_engine_scan(void *rs_lit_engine, int thread_id, } struct rs_lit_engine *rs_lit_inst = (struct rs_lit_engine *)rs_lit_engine; - struct rs_lit_stream *rs_lit_stream = rs_lit_inst->streams[thread_id]; + struct rs_lit_stream *rs_lit_stream = rs_lit_inst->per_thread_scratch_streams[thread_id]; assert(rs_lit_stream != NULL); utarray_clear(rs_lit_stream->matched_pat->pattern_ids); diff --git a/src/inc_internal/maat_redis_monitor.h b/src/inc_internal/maat_redis_monitor.h index bbdfa35..5b65c2a 100644 --- a/src/inc_internal/maat_redis_monitor.h +++ b/src/inc_internal/maat_redis_monitor.h @@ -26,6 +26,7 @@ extern "C" #include "uthash/uthash.h" struct foreign_key { + char *foreign_name; char *key; size_t key_len; char *filename; diff --git a/src/maat_plugin.c b/src/maat_plugin.c index 8929f50..6c30b04 100644 --- a/src/maat_plugin.c +++ b/src/maat_plugin.c @@ -222,9 +222,10 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema) } } -int plugin_table_get_foreign_names(struct plugin_schema *plugin_schema, - char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1]) +int plugin_table_get_foreign_names(struct plugin_schema *plugin_schema, char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1]) { + int n_foreign_keys = 0; + if (NULL == plugin_schema) { return -1; } @@ -233,7 +234,7 @@ int plugin_table_get_foreign_names(struct plugin_schema *plugin_schema, strncpy(foreign_names[i], plugin_schema->foreign_names[i], MAX_FOREIGN_NAME_LEN); } - return plugin_schema->n_foreign; + return n_foreign_keys; } int plugin_table_set_ex_container_schema(void *plugin_schema, int table_id, diff --git a/src/maat_redis_monitor.c b/src/maat_redis_monitor.c index 1e5fbf1..885927b 100644 --- a/src/maat_redis_monitor.c +++ b/src/maat_redis_monitor.c @@ -64,43 +64,65 @@ get_foreign_cont_filename(const char *table_name, const char* rule_uuid_str, } static void -get_foregin_keys(struct serial_rule *p_rule, const char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1], +get_foregin_keys(struct serial_rule *p_rule, char foreign_names[][MAX_FOREIGN_NAME_LEN + 1], int n_foreign, const char *dir, struct log_handle *logger) { int foreign_key_size = 0; p_rule->f_keys = ALLOC(struct foreign_key, n_foreign); for (int i = 0; i < n_foreign; i++) { - const char *p_foreign = foreign_names[i]; + const char *p_foreign_name = foreign_names[i]; + cJSON *json = cJSON_Parse(p_rule->table_line); - foreign_key_size = strlen(p_foreign); - //emtpy file - if (0 == strncasecmp(p_foreign, "null", strlen("null"))) { + if (json == NULL) { + log_fatal(logger, MODULE_REDIS_MONITOR, + "[%s:%d] Get %s,%s foreign key failed: " + "Invalid table line", __FUNCTION__, __LINE__, + p_rule->table_name, p_rule->rule_uuid_str); continue; } - if (0 != strncmp(p_foreign, foreign_source_prefix, + cJSON *item = cJSON_GetObjectItem(json, p_foreign_name); + if (item == NULL || item->type != cJSON_String) { + log_fatal(logger, MODULE_REDIS_MONITOR, + "[%s:%d] Get %s,%s foreign key failed: " + "Invalid foreign key %s", __FUNCTION__, __LINE__, + p_rule->table_name, p_rule->rule_uuid_str, p_foreign_name); + continue; + } + const char *p_foreign_key = item->valuestring; + + foreign_key_size = strlen(p_foreign_key); + //emtpy file + if (0 == strncasecmp(p_foreign_key, "null", strlen("null"))) { + continue; + } + + if (0 != strncmp(p_foreign_key, foreign_source_prefix, strlen(foreign_source_prefix))) { log_fatal(logger, MODULE_REDIS_MONITOR, "[%s:%d] Get %s,%s foreign key failed: " "Invalid source prefix %s", __FUNCTION__, __LINE__, - p_rule->table_name, p_rule->rule_uuid_str, p_foreign); + p_rule->table_name, p_rule->rule_uuid_str, p_foreign_key); continue; } foreign_key_size = foreign_key_size - strlen(foreign_source_prefix); - p_foreign += strlen(foreign_source_prefix); + p_foreign_key += strlen(foreign_source_prefix); - if (0 != strncmp(p_foreign, foreign_key_prefix, + if (0 != strncmp(p_foreign_key, foreign_key_prefix, strlen(foreign_key_prefix))) { log_info(logger, MODULE_REDIS_MONITOR, "[%s:%d] %s, %s foreign key prefix %s is not recommended", __FUNCTION__, __LINE__, p_rule->table_name, p_rule->rule_uuid_str, - p_foreign); + p_foreign_key); } + p_rule->f_keys[p_rule->n_foreign].foreign_name = ALLOC(char, strlen(p_foreign_name) + 1); + memcpy(p_rule->f_keys[p_rule->n_foreign].foreign_name, p_foreign_name, strlen(p_foreign_name)); + p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size + 1); - memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign, foreign_key_size); + memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign_key, foreign_key_size); p_rule->f_keys[p_rule->n_foreign].filename = get_foreign_cont_filename(p_rule->table_name, p_rule->rule_uuid_str, p_rule->f_keys[p_rule->n_foreign].key, dir); @@ -117,6 +139,7 @@ get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int rule_num, struct maat *maat_inst, const char *dir) { int rule_with_foreign_key = 0; + char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1] = {0}; for (int i = 0; i < rule_num; i++) { if (NULL == rule_list[i].table_line) { @@ -134,10 +157,8 @@ get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, continue; } - char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1]; int n_foreign = - plugin_table_get_foreign_names(schema, - foreign_names); + plugin_table_get_foreign_names(schema, foreign_names); if (0 == n_foreign) { continue; } @@ -158,7 +179,7 @@ int maat_get_foreign_keys_by_prefix(redisContext *ctx, int rule_with_foreign_key = 0; int n_foreign = 0; - char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1]; + char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1] = {0}; for (int i = 0; i < rule_num; i++) { n_foreign = 0; @@ -170,10 +191,9 @@ int maat_get_foreign_keys_by_prefix(redisContext *ctx, cJSON_ArrayForEach(item, json) { if (item->type == cJSON_String) { if (strlen(item->valuestring) > strlen(foreign_source_prefix) && - 0 == strncmp(item->valuestring, foreign_key_prefix, strlen(foreign_key_prefix))) { + 0 == strncmp(item->valuestring, foreign_source_prefix, strlen(foreign_source_prefix))) { - strncpy(foreign_names[n_foreign], item->valuestring, - MAX_FOREIGN_NAME_LEN); + strncpy(foreign_names[n_foreign], item->string, strlen(item->string)); n_foreign++; } } @@ -436,6 +456,7 @@ void maat_clear_rule_cache(struct serial_rule *s_rule) for (int i = 0; i < s_rule->n_foreign; i++) { FREE(s_rule->f_keys[i].filename); FREE(s_rule->f_keys[i].key); + FREE(s_rule->f_keys[i].foreign_name); } FREE(s_rule->f_keys); @@ -762,13 +783,13 @@ void maat_rewrite_table_line_with_foreign(struct serial_rule *s_rule) } for (i = 0; i < s_rule->n_foreign; i++) { - cJSON *tmp_obj = cJSON_GetObjectItem(json, s_rule->f_keys[i].key); + cJSON *tmp_obj = cJSON_GetObjectItem(json, s_rule->f_keys[i].foreign_name); if (NULL == tmp_obj) { continue; } - cJSON_DeleteItemFromObject(json, s_rule->f_keys[i].key); - cJSON_AddStringToObject(json, s_rule->f_keys[i].key, s_rule->f_keys[i].filename); + cJSON_DeleteItemFromObject(json, s_rule->f_keys[i].foreign_name); + cJSON_AddStringToObject(json, s_rule->f_keys[i].foreign_name, s_rule->f_keys[i].filename); } char *rewrite_line_json = cJSON_PrintUnformatted(json); diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 2fe19a5..18caeb9 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -487,6 +487,7 @@ protected: maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); maat_options_set_hit_path_enabled(opts); + maat_options_set_hit_object_enabled(opts); maat_options_set_expr_engine(opts, MAAT_EXPR_ENGINE_HS); _shared_maat_inst = maat_new(opts, g_table_info_path); @@ -1189,6 +1190,54 @@ TEST_F(HsStringScan, StreamInput) { state = NULL; } +TEST_F(HsStringScan, StreamHitDirectObject) { + uuid_t results[ARRAY_SIZE]; + size_t n_hit_result = 0; + int thread_id = 0; + int ret; + struct maat *maat_inst = HsStringScan::_shared_maat_inst; + struct maat_state *state = maat_state_new(maat_inst, thread_id); + const char *table_name = "HTTP_URL"; + const char *attribute_name = "HTTP_URL"; + const char *scan_data1 = "www.3300av.com"; + const char *scan_data2 = "sdadhuadhasdgufgh;sdfhjaufhiwebfiusdafhaos;dhfaluhjweh"; + + memset(results, 0, sizeof(results)); + + struct maat_stream *sp = maat_stream_new(maat_inst, table_name, attribute_name, state); + ASSERT_TRUE(sp != NULL); + + ret = maat_stream_scan(sp, scan_data1, strlen(scan_data1), results, + ARRAY_SIZE, &n_hit_result, state); + + EXPECT_EQ(ret, MAAT_SCAN_HIT); + char uuid_str[UUID_STR_LEN] = {0}; + uuid_unparse(results[0], uuid_str); + EXPECT_STREQ(uuid_str, "00000000-0000-0000-0000-000000000133"); + + struct maat_hit_object object_array[ARRAY_SIZE]; + ret = maat_state_get_direct_hit_objects(state, object_array, ARRAY_SIZE); + EXPECT_EQ(ret, 1); + uuid_unparse(object_array[0].object_uuid, uuid_str); + EXPECT_STREQ(uuid_str, "00000000-0000-0000-0000-000000000112"); + + ret = maat_scan_not_logic(maat_inst, table_name, attribute_name, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_stream_scan(sp, scan_data2, strlen(scan_data2), results, + ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + ret = maat_state_get_direct_hit_objects(state, object_array, ARRAY_SIZE); + EXPECT_EQ(ret, 0); + + maat_stream_free(sp); + + + maat_state_free(state); + state = NULL; +} + #if 0 //TODO TEST_F(HsStringScan, dynamic_config) { const char *table_name = "HTTP_URL"; diff --git a/test/table_info.json b/test/table_info.json index b11b136..5d0cf68 100644 --- a/test/table_info.json +++ b/test/table_info.json @@ -62,7 +62,6 @@ "table_name":"HTTP_REGION", "db_tables":["HTTP_URL", "HTTP_HOST"], "table_type":"expr", - "expr_engine":"rulescan", "schema_tag": "{\"http_region\": \"expr\"}" }, {