diff --git a/src/maat_api.c b/src/maat_api.c index ba24cf9..21b9f86 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -1654,8 +1654,9 @@ int maat_state_set_scan_compile_tables(struct maat *maat_instance, struct maat_s } struct maat_state *mid = grab_state(state, maat_instance, -1); + mid->n_compile_table = n_table; + for (size_t i = 0; i < mid->n_compile_table; i++) { - strncpy(mid->compile_tables[i], compile_tables[i], strlen(compile_tables[i])); } diff --git a/src/maat_compile.c b/src/maat_compile.c index 01ab2fe..7fbef39 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -28,23 +28,17 @@ #define MODULE_COMPILE module_name_str("maat.compile") #define MAX_TABLE_LINE_SIZE (1024 * 16) -enum user_region_encode { - USER_REGION_ENCODE_NONE = 0, - USER_REGION_ENCODE_ESCAPE, - USER_REGION_ENCODE_BASE64 -}; - struct compile_schema { int compile_id_column; - int tags_column; - int user_region_column; + int rule_tag_column; int declared_clause_num_column; int evaluation_order_column; - enum user_region_encode user_region_encoding; struct ex_data_schema *ex_schema; int table_id; //ugly struct table_manager *ref_tbl_mgr; - size_t unmatched_tag_cnt; + + unsigned long long update_err_cnt; + unsigned long long unmatch_tag_cnt; }; struct group2compile_schema { @@ -60,7 +54,6 @@ struct group2compile_schema { struct compile_item { long long compile_id; - char user_region[MAX_TABLE_LINE_SIZE]; int declared_clause_num; int evaluation_order; }; @@ -303,20 +296,6 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } - item = cJSON_GetObjectItem(json, "user_region_encoded"); - if (item != NULL && item->type == cJSON_String) { - if (strcmp(item->valuestring, "escape") == 0) { - compile_schema->user_region_encoding = USER_REGION_ENCODE_ESCAPE; - } else if (strcmp(item->valuestring, "none") == 0) { - compile_schema->user_region_encoding = USER_REGION_ENCODE_NONE; - } else { - log_error(logger, MODULE_COMPILE, - "table %s has no user_region_encoded column", table_name); - goto error; - } - read_cnt++; - } - item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { log_error(logger, MODULE_COMPILE, @@ -332,13 +311,7 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, custom_item = cJSON_GetObjectItem(item, "tags"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - compile_schema->tags_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "user_region"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - compile_schema->user_region_column = custom_item->valueint; + compile_schema->rule_tag_column = custom_item->valueint; read_cnt++; } @@ -356,7 +329,7 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, compile_schema->ref_tbl_mgr = tbl_mgr; - if (read_cnt < 7) { + if (read_cnt < 5) { goto error; } @@ -450,17 +423,61 @@ int group2compile_associated_compile_table_id(void *g2c_schema) return schema->associated_compile_table_id; } +int compile_accept_tag_match(struct compile_schema *schema, const char *line, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr); + + if (schema->rule_tag_column > 0 && n_tag > 0) { + int ret = get_column_pos(line, schema->rule_tag_column, + &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) has no rule_tag, line:%s", + schema->table_id, line); + schema->update_err_cnt++; + return TAG_MATCH_ERR; + } + + if (column_len > 2) { + char *tag_str = ALLOC(char, column_len + 1); + memcpy(tag_str, (line + column_offset), column_len); + ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str); + FREE(tag_str); + if (TAG_MATCH_ERR == ret) { + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) has invalid tag format, line:%s", + schema->table_id, line); + schema->update_err_cnt++; + return TAG_MATCH_ERR; + } + + if (TAG_MATCH_UNMATCHED == ret) { + schema->unmatch_tag_cnt++; + return TAG_MATCH_UNMATCHED; + } + } + } + + return TAG_MATCH_MATCHED; +} + struct compile_item * compile_item_new(const char *line, struct compile_schema *compile_schema, struct log_handle *logger) { + int ret = compile_accept_tag_match(compile_schema, line, logger); + if (ret == TAG_MATCH_UNMATCHED) { + return NULL; + } + size_t column_offset = 0; size_t column_len = 0; - char tag_str[MAX_TABLE_LINE_SIZE] = {0}; - size_t n_accept_tag = 0; struct compile_item *compile_item = ALLOC(struct compile_item, 1); - int ret = get_column_pos(line, compile_schema->compile_id_column, + ret = get_column_pos(line, compile_schema->compile_id_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_COMPILE, @@ -470,60 +487,6 @@ compile_item_new(const char *line, struct compile_schema *compile_schema, } compile_item->compile_id = atoll(line + column_offset); - ret = get_column_pos(line, compile_schema->tags_column, - &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_COMPILE, - "compile table(table_id:%d) line:%s has no tags", - compile_schema->table_id, line); - goto error; - } - - memcpy(tag_str, (line + column_offset), column_len); - n_accept_tag = table_manager_accept_tags_count(compile_schema->ref_tbl_mgr); - - if (n_accept_tag > 0 && strlen(tag_str) > 2) { - str_unescape(tag_str); - ret = table_manager_accept_tags_match(compile_schema->ref_tbl_mgr, tag_str); - if (TAG_MATCH_ERR == ret) { - log_error(logger, MODULE_COMPILE, - "compile table(table_id:%d) line:%s is invalid tag", - compile_schema->table_id, line); - goto error; - } - - if (TAG_MATCH_MATCHED == ret) { //not matched - compile_schema->unmatched_tag_cnt++; - goto error; - } - } - - ret = get_column_pos(line, compile_schema->user_region_column, - &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_COMPILE, - "compile table(table_id:%d) line:%s has no user_region", - compile_schema->table_id, line); - goto error; - } - - if (column_len > MAX_TABLE_LINE_SIZE) { - log_error(logger, MODULE_COMPILE, - "compile table(table_id:%d) line:%s user_region too long", - compile_schema->table_id, line); - goto error; - } - - memcpy(compile_item->user_region, (line + column_offset), column_len); - - switch (compile_schema->user_region_encoding) { - case USER_REGION_ENCODE_ESCAPE: - str_unescape(compile_item->user_region); - break; - default: - break; - } - ret = get_column_pos(line, compile_schema->declared_clause_num_column, &column_offset, &column_len); if (ret < 0) { diff --git a/src/maat_flag.c b/src/maat_flag.c index ba35233..3751607 100644 --- a/src/maat_flag.c +++ b/src/maat_flag.c @@ -34,10 +34,10 @@ struct flag_schema { }; struct flag_item { - uint64_t item_id; - uint64_t group_id; - uint64_t flag; - uint64_t flag_mask; + long long item_id; + long long group_id; + long long flag; + long long flag_mask; }; struct flag_runtime { @@ -176,7 +176,7 @@ void flag_runtime_free(void *flag_runtime) } int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len, - uint64_t item_id, struct flag_rule *rule, int is_valid) + long long item_id, struct flag_rule *rule, int is_valid) { void *data = NULL; @@ -301,7 +301,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, return -1; } else if (0 == is_valid) { //delete - HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(uint64_t), item); + HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(long long), item); if (NULL == item) { return -1; } @@ -317,7 +317,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free); } else { //add - HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(uint64_t), item); + HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(long long), item); if (item) { log_error(flag_rt->logger, MODULE_FLAG, "flag runtime add item %d to item_hash failed, already exist", @@ -332,7 +332,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, u_para = maat_item_inner_new(flag_item->group_id, item_id, 0); item = maat_item_new(item_id, flag_item->group_id, u_para); - HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(uint64_t), item); + HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(long long), item); flag_rule = flag_item_to_flag_rule(flag_item, u_para); flag_item_free(flag_item); @@ -345,7 +345,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, } char *key = (char *)&item_id; - int ret = flag_runtime_update_row(flag_rt, key, sizeof(uint64_t), item_id, flag_rule, is_valid); + int ret = flag_runtime_update_row(flag_rt, key, sizeof(long long), item_id, flag_rule, is_valid); if (ret < 0) { if (flag_rule != NULL) { flag_rule_free(flag_rule); @@ -457,4 +457,4 @@ void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id) long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread) { return alignment_int64_array_sum(flag_rt->hit_cnt, n_thread); -} \ No newline at end of file +} diff --git a/src/maat_plugin.c b/src/maat_plugin.c index c4fc3ff..faf7698 100644 --- a/src/maat_plugin.c +++ b/src/maat_plugin.c @@ -285,9 +285,6 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, return -1; } plugin_rt->acc_line_num++; - log_info(plugin_rt->logger, MODULE_PLUGIN, - " add ex_data_runtime, key:%s key_len:%zu line:%s", - key, key_len, row); } } @@ -301,8 +298,6 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, if ((NULL == ex_schema) && (0 == cb_count)) { ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row); - log_info(plugin_rt->logger, MODULE_PLUGIN, - " cache_row_put, line:%s", row); } return 0; diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index ccc7446..3f0008f 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -1230,6 +1230,91 @@ TEST_F(CompileTable, CompileRuleUpdate) { EXPECT_EQ(ret, 1); } +class Policy : public testing::Test +{ +protected: + static void SetUpTestCase() { + + } + + static void TearDownTestCase() { + + } +}; + +void accept_tags_entry_cb(int table_id, const char *table_line, void *u_para) +{ + int* callback_times = (int *)u_para; + char status[32] = {0}; + int entry_id = -1, seq = -1; + int is_valid = 0; + + sscanf(table_line, "%d\t%s\t%d\t%d", &seq,status, &entry_id, &is_valid); + EXPECT_STREQ(status, "SUCCESS"); + (*callback_times)++; +} + +TEST_F(Policy, PluginRuleTags1) { + const char *table_name = "TEST_EFFECTIVE_RANGE_TABLE"; + int table_id = maat_table_get_id(g_maat_instance, table_name); + ASSERT_GT(table_id, 0); + + int callback_times=0; + int ret = maat_table_callback_register(g_maat_instance, table_id, + NULL, + accept_tags_entry_cb, + NULL, + &callback_times); + ASSERT_GE(ret, 0); + EXPECT_EQ(callback_times, 5); +} + +void accept_tags_entry2_cb(int table_id, const char *table_line, void *u_para) +{ + int *callback_times = (int *)u_para; + (*callback_times)++; +} + +TEST_F(Policy, PluginRuleTags2) { + const char *table_name = "IR_INTERCEPT_IP"; + int table_id = maat_table_get_id(g_maat_instance, table_name); + ASSERT_GT(table_id, 0); + + int callback_times = 0; + int ret = maat_table_callback_register(g_maat_instance, table_id, + NULL, + accept_tags_entry2_cb, + NULL, + &callback_times); + ASSERT_GE(ret, 0); + EXPECT_EQ(callback_times, 2); +} + +TEST_F(Policy, CompileRuleTags) { + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + const char *should_hit = "string bbb should hit"; + const char *should_not_hit = "string aaa should not hit"; + const char *table_name = "HTTP_URL_LITERAL"; + + int table_id = maat_table_get_id(g_maat_instance, table_name); + ASSERT_GT(table_id, 0); + + int ret = maat_scan_string(g_maat_instance, table_id, 0, should_not_hit, + strlen(should_not_hit), results, ARRAY_SIZE, + &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_string(g_maat_instance, table_id, 0, should_hit, + strlen(should_hit), results, ARRAY_SIZE, + &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + + maat_state_free(&state); +} + + class MaatCmdTest : public testing::Test { protected: @@ -1480,6 +1565,7 @@ int main(int argc, char ** argv) return -1; } + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; char redis_ip[64] = "127.0.0.1"; int redis_port = 6379; int redis_db = 0; @@ -1494,6 +1580,7 @@ int main(int argc, char ** argv) struct maat_options *opts = maat_options_new(); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_logger(opts, logger); + maat_options_set_accept_tags(opts, accept_tags); g_maat_instance = maat_new(opts, table_info_path); maat_options_free(opts); diff --git a/test/table_info.conf b/test/table_info.conf index dda8f97..d243da4 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -3,12 +3,10 @@ "table_id":0, "table_name":"COMPILE", "table_type":"compile", - "user_region_encoded":"escape", "valid_column":8, "custom": { "compile_id":1, "tags":6, - "user_region":7, "clause_num":9, "evaluation_order":10 }