fix compile conjunction bug

src/maat_api.c

@@ -1654,8 +1654,9 @@ int maat_state_set_scan_compile_tables(struct maat *maat_instance, struct maat_s
     }
 
     struct maat_state *mid = grab_state(state, maat_instance, -1);
-    for (size_t i = 0; i < mid->n_compile_table; i++) {
+    mid->n_compile_table = n_table;
 
+    for (size_t i = 0; i < mid->n_compile_table; i++) {
         strncpy(mid->compile_tables[i], compile_tables[i], strlen(compile_tables[i]));
     }
 

src/maat_compile.c

@@ -28,23 +28,17 @@
 #define MODULE_COMPILE	 module_name_str("maat.compile")
 #define MAX_TABLE_LINE_SIZE  (1024 * 16)
 
-enum user_region_encode {
-	USER_REGION_ENCODE_NONE = 0,
-	USER_REGION_ENCODE_ESCAPE,
-	USER_REGION_ENCODE_BASE64
-};
-
 struct compile_schema {
     int compile_id_column;
-    int tags_column;
+    int rule_tag_column;
-    int user_region_column;
     int declared_clause_num_column;
     int evaluation_order_column;
-    enum user_region_encode user_region_encoding;
 	struct ex_data_schema *ex_schema;
     int table_id; //ugly
     struct table_manager *ref_tbl_mgr;
-    size_t unmatched_tag_cnt;
+
+    unsigned long long update_err_cnt;
+    unsigned long long unmatch_tag_cnt;
 };
 
 struct group2compile_schema {
@@ -60,7 +54,6 @@ struct group2compile_schema {
 
 struct compile_item {
     long long compile_id;
-    char user_region[MAX_TABLE_LINE_SIZE];
     int declared_clause_num;
     int evaluation_order;
 };
@@ -303,20 +296,6 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         read_cnt++;
     }
 
-    item = cJSON_GetObjectItem(json, "user_region_encoded");
-    if (item != NULL && item->type == cJSON_String) {
-        if (strcmp(item->valuestring, "escape") == 0) {
-            compile_schema->user_region_encoding = USER_REGION_ENCODE_ESCAPE;
-        } else if (strcmp(item->valuestring, "none") == 0) {
-            compile_schema->user_region_encoding = USER_REGION_ENCODE_NONE;
-        } else {
-            log_error(logger, MODULE_COMPILE,
-                      "table %s has no user_region_encoded column", table_name);
-            goto error;
-        }
-        read_cnt++;
-    }
-
    	item = cJSON_GetObjectItem(json, "custom");
     if (item == NULL || item->type != cJSON_Object) {
         log_error(logger, MODULE_COMPILE, 
@@ -332,13 +311,7 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
 
     custom_item = cJSON_GetObjectItem(item, "tags");
     if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        compile_schema->tags_column = custom_item->valueint;
+        compile_schema->rule_tag_column = custom_item->valueint;
-        read_cnt++;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "user_region");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        compile_schema->user_region_column = custom_item->valueint;
         read_cnt++;
     }
 
@@ -356,7 +329,7 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
 
     compile_schema->ref_tbl_mgr = tbl_mgr;
 
-    if (read_cnt < 7) {
+    if (read_cnt < 5) {
 		goto error;
     }
 
@@ -450,17 +423,61 @@ int group2compile_associated_compile_table_id(void *g2c_schema)
     return schema->associated_compile_table_id;
 }
 
+int compile_accept_tag_match(struct compile_schema *schema, const char *line,
+                             struct log_handle *logger)
+{
+    size_t column_offset = 0;
+    size_t column_len = 0;
+    size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr);
+
+    if (schema->rule_tag_column > 0 && n_tag > 0) {
+        int ret = get_column_pos(line, schema->rule_tag_column, 
+                                 &column_offset, &column_len);
+        if (ret < 0) {
+            log_error(logger, MODULE_COMPILE,
+                      "compile table(table_id:%d) has no rule_tag, line:%s",
+                      schema->table_id, line);
+            schema->update_err_cnt++;
+            return TAG_MATCH_ERR;
+        }
+
+        if (column_len > 2) {
+            char *tag_str = ALLOC(char, column_len + 1);
+            memcpy(tag_str, (line + column_offset), column_len);
+            ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str);
+            FREE(tag_str);
+            if (TAG_MATCH_ERR == ret) {
+                log_error(logger, MODULE_COMPILE,
+                          "compile table(table_id:%d) has invalid tag format, line:%s",
+                          schema->table_id, line);
+                schema->update_err_cnt++;
+                return TAG_MATCH_ERR;
+            }
+
+            if (TAG_MATCH_UNMATCHED == ret) {
+                schema->unmatch_tag_cnt++;
+                return TAG_MATCH_UNMATCHED;
+            }
+        }
+    }
+
+    return TAG_MATCH_MATCHED;
+}
+
 struct compile_item *
 compile_item_new(const char *line, struct compile_schema *compile_schema,
                  struct log_handle *logger)
 {
+    int ret = compile_accept_tag_match(compile_schema, line, logger);
+    if (ret == TAG_MATCH_UNMATCHED) {
+        return NULL;
+    }
+
     size_t column_offset = 0;
     size_t column_len = 0;
-    char tag_str[MAX_TABLE_LINE_SIZE] = {0};
-    size_t n_accept_tag = 0;
     struct compile_item *compile_item = ALLOC(struct compile_item, 1);
 
-    int ret = get_column_pos(line, compile_schema->compile_id_column, 
+    ret = get_column_pos(line, compile_schema->compile_id_column, 
                              &column_offset, &column_len);
     if (ret < 0) {
         log_error(logger, MODULE_COMPILE, 
@@ -470,60 +487,6 @@ compile_item_new(const char *line, struct compile_schema *compile_schema,
     }
     compile_item->compile_id = atoll(line + column_offset);
 
-    ret = get_column_pos(line, compile_schema->tags_column,
-                         &column_offset, &column_len);
-    if (ret < 0) {
-        log_error(logger, MODULE_COMPILE, 
-                  "compile table(table_id:%d) line:%s has no tags",
-                  compile_schema->table_id, line);
-        goto error;
-    }
-    
-    memcpy(tag_str, (line + column_offset), column_len);
-    n_accept_tag = table_manager_accept_tags_count(compile_schema->ref_tbl_mgr);
-
-    if (n_accept_tag > 0 && strlen(tag_str) > 2) {
-		str_unescape(tag_str);
-        ret = table_manager_accept_tags_match(compile_schema->ref_tbl_mgr, tag_str);
-		if (TAG_MATCH_ERR == ret) {
-			log_error(logger, MODULE_COMPILE,
-					  "compile table(table_id:%d) line:%s is invalid tag",
-					  compile_schema->table_id, line);
-			goto error;
-		}
-
-		if (TAG_MATCH_MATCHED == ret) { //not matched
-			compile_schema->unmatched_tag_cnt++;
-            goto error;
-		}
-	}
-
-    ret = get_column_pos(line, compile_schema->user_region_column, 
-                         &column_offset, &column_len);
-    if (ret < 0) {
-        log_error(logger, MODULE_COMPILE,
-				  "compile table(table_id:%d) line:%s has no user_region",
-					  compile_schema->table_id, line);
-		goto error;
-    }
-
-    if (column_len > MAX_TABLE_LINE_SIZE) {
-        log_error(logger, MODULE_COMPILE,
-				  "compile table(table_id:%d) line:%s user_region too long",
-				  compile_schema->table_id, line);
-		goto error;
-    }
-    
-    memcpy(compile_item->user_region, (line + column_offset), column_len);
-
-    switch (compile_schema->user_region_encoding) {
-		case USER_REGION_ENCODE_ESCAPE:
-			str_unescape(compile_item->user_region);
-			break;
-		default:
-			break;
-	}
-
     ret = get_column_pos(line, compile_schema->declared_clause_num_column, 
                          &column_offset, &column_len);
     if (ret < 0) {

src/maat_flag.c

@@ -34,10 +34,10 @@ struct flag_schema {
 };
 
 struct flag_item {
-    uint64_t item_id;
+    long long item_id;
-    uint64_t group_id;
+    long long group_id;
-    uint64_t flag;
+    long long flag;
-    uint64_t flag_mask;
+    long long flag_mask;
 };
 
 struct flag_runtime {
@@ -176,7 +176,7 @@ void flag_runtime_free(void *flag_runtime)
 }
 
 int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len,
-                            uint64_t item_id, struct flag_rule *rule, int is_valid)
+                            long long item_id, struct flag_rule *rule, int is_valid)
 {
     void *data = NULL;
 
@@ -301,7 +301,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
         return -1;
     } else if (0 == is_valid) {
         //delete
-        HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(uint64_t), item);
+        HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(long long), item);
         if (NULL == item) {
             return -1;
         }
@@ -317,7 +317,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
         maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free);
     } else {
         //add
-        HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(uint64_t), item);
+        HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(long long), item);
         if (item) {
             log_error(flag_rt->logger, MODULE_FLAG, 
                       "flag runtime add item %d to item_hash failed, already exist", 
@@ -332,7 +332,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
 
         u_para = maat_item_inner_new(flag_item->group_id, item_id, 0);
         item = maat_item_new(item_id, flag_item->group_id, u_para);
-        HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(uint64_t), item);
+        HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(long long), item);
 
         flag_rule = flag_item_to_flag_rule(flag_item, u_para);
         flag_item_free(flag_item);
@@ -345,7 +345,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
     }
 
     char *key = (char *)&item_id;
-    int ret = flag_runtime_update_row(flag_rt, key, sizeof(uint64_t), item_id, flag_rule, is_valid);
+    int ret = flag_runtime_update_row(flag_rt, key, sizeof(long long), item_id, flag_rule, is_valid);
     if (ret < 0) {
         if (flag_rule != NULL) {
             flag_rule_free(flag_rule);

src/maat_plugin.c

@@ -285,9 +285,6 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
                 return -1;
             }
             plugin_rt->acc_line_num++;
-            log_info(plugin_rt->logger, MODULE_PLUGIN, 
-                 "<plugin_runtime_update> add ex_data_runtime, key:%s key_len:%zu line:%s", 
-                 key, key_len, row);
         }
     }
 
@@ -301,8 +298,6 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
     
     if ((NULL == ex_schema) && (0 == cb_count)) {
         ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row);
-        log_info(plugin_rt->logger, MODULE_PLUGIN, 
-                 "<plugin_runtime_update> cache_row_put, line:%s", row);
     }
 
     return 0;

test/maat_framework_gtest.cpp

@@ -1230,6 +1230,91 @@ TEST_F(CompileTable, CompileRuleUpdate) {
     EXPECT_EQ(ret, 1);
 }
 
+class Policy : public testing::Test
+{
+protected:
+    static void SetUpTestCase() {
+        
+    }
+
+    static void TearDownTestCase() {
+       
+    }    
+};
+
+void accept_tags_entry_cb(int table_id, const char *table_line, void *u_para)
+{
+	int* callback_times = (int *)u_para;
+	char status[32] = {0};
+	int entry_id = -1, seq = -1;
+	int is_valid = 0;
+
+	sscanf(table_line, "%d\t%s\t%d\t%d", &seq,status, &entry_id, &is_valid);
+	EXPECT_STREQ(status, "SUCCESS");
+	(*callback_times)++;
+}
+
+TEST_F(Policy, PluginRuleTags1) {
+    const char *table_name = "TEST_EFFECTIVE_RANGE_TABLE";
+    int table_id = maat_table_get_id(g_maat_instance, table_name);
+    ASSERT_GT(table_id, 0);
+
+    int callback_times=0;
+	int ret = maat_table_callback_register(g_maat_instance, table_id,
+							               NULL,
+							               accept_tags_entry_cb, 
+							               NULL,
+							               &callback_times);
+	ASSERT_GE(ret, 0);
+	EXPECT_EQ(callback_times, 5);
+}
+
+void accept_tags_entry2_cb(int table_id, const char *table_line, void *u_para)
+{
+	int *callback_times = (int *)u_para;
+	(*callback_times)++;
+}
+
+TEST_F(Policy, PluginRuleTags2) {
+    const char *table_name = "IR_INTERCEPT_IP";
+	int table_id = maat_table_get_id(g_maat_instance, table_name);
+	ASSERT_GT(table_id, 0);
+
+	int callback_times = 0;
+	int ret = maat_table_callback_register(g_maat_instance, table_id,
+							               NULL,
+							               accept_tags_entry2_cb, 
+							               NULL,
+							               &callback_times);
+	ASSERT_GE(ret, 0);
+	EXPECT_EQ(callback_times, 2);
+}
+
+TEST_F(Policy, CompileRuleTags) {
+    long long results[ARRAY_SIZE] = {0};
+    size_t n_hit_result = 0;
+    struct maat_state *state = NULL;
+	const char *should_hit = "string bbb should hit";
+	const char *should_not_hit = "string aaa should not hit";
+	const char *table_name = "HTTP_URL_LITERAL";
+	
+    int table_id = maat_table_get_id(g_maat_instance, table_name);
+	ASSERT_GT(table_id, 0);
+
+	int ret = maat_scan_string(g_maat_instance, table_id, 0, should_not_hit, 
+                               strlen(should_not_hit), results, ARRAY_SIZE,
+						        &n_hit_result, &state);
+	EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
+	
+	ret = maat_scan_string(g_maat_instance, table_id, 0, should_hit, 
+                           strlen(should_hit), results, ARRAY_SIZE,
+						   &n_hit_result, &state);
+	EXPECT_EQ(ret, MAAT_SCAN_HIT);
+
+	maat_state_free(&state);
+}
+
+
 class MaatCmdTest : public testing::Test
 {
 protected:
@@ -1480,6 +1565,7 @@ int main(int argc, char ** argv)
         return -1;
     }
 
+    const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
     char redis_ip[64] = "127.0.0.1";
     int redis_port = 6379;
     int redis_db = 0;
@@ -1494,6 +1580,7 @@ int main(int argc, char ** argv)
     struct maat_options *opts = maat_options_new();
     maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
     maat_options_set_logger(opts, logger);
+    maat_options_set_accept_tags(opts, accept_tags);
 
     g_maat_instance = maat_new(opts, table_info_path);
     maat_options_free(opts);

test/table_info.conf

@@ -3,12 +3,10 @@
         "table_id":0,
         "table_name":"COMPILE",
         "table_type":"compile",
-        "user_region_encoded":"escape",
         "valid_column":8,
         "custom": {
             "compile_id":1,
             "tags":6,
-            "user_region":7,
             "clause_num":9,
             "evaluation_order":10
         }