gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add json/redis rule parser

Browse Source
This commit is contained in:
liuwentan
2022-12-03 22:23:41 +08:00
parent 84a271144b
commit ea4c1ba4c3
32 changed files with 6734 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/inc_internal/json2iris.h Normal file
View File

@@ -0,0 +1,29 @@
/*
**********************************************************************************************
* File: json2iris.h
* Description: rule for transform json2iris
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _JSON2IRIS_H_
#define _JSON2IRIS_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "hiredis/hiredis.h"
int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn,
const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx,
char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo);
#ifdef __cpluscplus
}
#endif
#endif

39
src/inc_internal/maat_command.h Normal file
View File

@@ -0,0 +1,39 @@
/*
**********************************************************************************************
* File: maat_command.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_COMMAND_H_
#define _MAAT_COMMAND_H_
#ifdef __cpluscplus
extern "C"
{
#endif
enum maat_operation {
MAAT_OP_DEL = 0,
MAAT_OP_ADD,
MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after
};
struct maat_cmd_line
{
const char *table_name;
const char *table_line;
int rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary.
int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout.
};
int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule);
#ifdef __cpluscplus
}
#endif
#endif

9
src/inc_internal/maat_common.h
View File

@@ -18,6 +18,8 @@ extern "C"
#include <stddef.h>
#include "maat_rule.h"
struct maat_options {
size_t nr_worker_threads;
int rule_effect_interval_ms;
@@ -25,8 +27,11 @@ struct maat_options {
int gc_timeout_ms;
int deferred_load_on;
enum data_source input_mode;
char iris_full_dir[NAME_MAX];
char iris_inc_dir[NAME_MAX];
union {
struct source_iris_ctx iris_ctx;
struct source_json_ctx json_ctx;
struct source_redis_ctx redis_ctx;
};
};
#ifdef __cpluscplus

2
src/inc_internal/maat_config_monitor.h
View File

@@ -28,6 +28,8 @@ void config_monitor_traverse(long long version, const char *idx_dir,
void (*finish_fn)(void *),
void *u_param);
int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz);
#ifdef __cpluscplus
}
#endif

33
src/inc_internal/maat_redis_monitor.h Normal file
View File

@@ -0,0 +1,33 @@
/*
**********************************************************************************************
* File: maat_redis_monitor.h
* Description: maat redis monitor api
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-11-29
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_REDIS_MONITOR_H_
#define _MAAT_REDIS_MONITOR_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "maat_rule.h"
#include <stdint.h>
void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx,
void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *),
void *u_param);
#ifdef __cpluscplus
}
#endif
#endif

110
src/inc_internal/maat_rule.h
View File

@@ -21,6 +21,13 @@ extern "C"
#include <limits.h>
#include <sys/time.h>
#include <pthread.h>
#include <sys/queue.h>
#include <openssl/md5.h>
#include "hiredis/hiredis.h"
#include "uthash/uthash.h"
#include "maat_table_schema.h"
#include "maat_command.h"
struct maat_runtime {
/* maat_runtime can be created and destroy dynamic, so need version info */
@@ -37,7 +44,9 @@ struct maat_runtime {
enum data_source {
DATA_SOURCE_NONE = 0,
DATA_SOURCE_IRIS_FILE
DATA_SOURCE_REDIS,
DATA_SOURCE_IRIS_FILE,
DATA_SOURCE_JSON_FILE
};
struct source_iris_ctx {
@@ -45,6 +54,52 @@ struct source_iris_ctx {
char full_dir[NAME_MAX];
};
struct source_json_ctx
{
char json_file[NAME_MAX];
char iris_file[NAME_MAX];
char effective_json_md5[MD5_DIGEST_LENGTH*2+1];
struct timespec last_md5_time;
};
struct source_redis_ctx
{
redisContext *read_ctx;
redisContext *write_ctx;
char redis_ip[64];
uint16_t redis_port;
int redis_db;
time_t last_reconnect_time;
};
struct foreign_key {
int column;
char *key;
size_t key_len;
char *filename;
};
//rm= Redis Maat
struct serial_rule {
enum maat_operation op;//0: delete, 1: add.
unsigned long rule_id;
int label_id;
long long timeout; // absolute unix time.
char table_name[NAME_MAX];
char *table_line;
int n_foreign;
struct foreign_key *f_keys;
TAILQ_ENTRY(serial_rule) entries;
UT_hash_handle hh;
};
#define POSSIBLE_REDIS_REPLY_SIZE 2
struct expected_reply {
int s_rule_seq;
int possible_reply_num;
redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE];
};
struct maat {
char instance_name[NAME_MAX];
@@ -56,6 +111,8 @@ struct maat {
enum data_source input_mode;
union {
struct source_iris_ctx iris_ctx;
struct source_json_ctx json_ctx;
struct source_redis_ctx mr_ctx;
};
int deferred_load;
@@ -72,7 +129,24 @@ struct maat {
int rule_update_checking_interval_ms;
int gc_timeout_ms; //garbage collection timeout_ms;
int cumulative_update_off; //Default: cumulative update on
struct maat_garbage_bin *garbage_bin;
char compile_tn[NAME_MAX];
char group_tn[NAME_MAX];
char group2compile_tn[NAME_MAX];
char group2group_tn[NAME_MAX];
char decrypt_key[NAME_MAX];
char decrypt_algo[NAME_MAX];
int maat_json_is_gzipped;
long long load_specific_version; //Default: Load the Latest. Only valid in redis mode, and maybe failed for too old
char foreign_cont_dir[NAME_MAX];
/* statistics */
long long line_cmd_acc_num;
};
void maat_start_cb(long long new_version, int update_type, void *u_para);
@@ -85,6 +159,40 @@ void *rule_monitor_loop(void *arg);
void maat_read_full_config(struct maat *maat_instance);
/* maat command API for internal */
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db);
redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...);
int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply);
long long maat_cmd_redis_server_time_s(redisContext *c);
long long maat_cmd_read_redis_integer(const redisReply *reply);
int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq);
const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len);
int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time);
void maat_cmd_empty_serial_rule(struct serial_rule *s_rule);
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version,
long long *new_version, struct table_schema_manager* table_schema_mgr,
struct serial_rule **list, int *update_type, int cumulative_off);
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process);
int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num, const char* dir);
void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn);
void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule);
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id,
const char *table_name, const char *line, long long timeout);
#ifdef __cpluscplus
}
#endif

45
src/inc_internal/maat_table_schema.h
View File

@@ -25,13 +25,28 @@ extern "C"
#define MAX_DISTRICT_STR 128
#define MAX_IP_STR 128
#define MAX_KEYWORDS_STR 1024
#define MAX_FOREIGN_CLMN_NUM 8
enum table_type {
TABLE_TYPE_EXPR = 0,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_IP,
TABLE_TYPE_IP_PLUS,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_INTERVAL_PLUS,
TABLE_TYPE_DIGEST,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_FQDN_PLUGIN,
TABLE_TYPE_BOOL_PLUGIN,
//above are physical table
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_COMPOSITION,
TABLE_TYPE_COMPILE,
TABLE_TYPE_GROUP,
TABLE_TYPE_GROUP2GROUP,
TABLE_TYPE_GROUP2COMPILE,
TABLE_TYPE_MAX
};
@@ -42,8 +57,21 @@ enum expr_type {
EXPR_TYPE_MAX
};
enum scan_type {
SCAN_TYPE_INVALID = -1,
SCAN_TYPE_NONE = 0,
SCAN_TYPE_PLUGIN,
SCAN_TYPE_IP_PLUGIN,
SCAN_TYPE_FQDN_PLUGIN,
SCAN_TYPE_BOOL_PLUGIN,
SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING,
SCAN_TYPE_MAX
};
enum match_method {
MATCH_METHOD_SUB=0,
MATCH_METHOD_SUB = 0,
MATCH_METHOD_RIGHT,
MATCH_METHOD_LEFT,
MATCH_METHOD_COMPLETE,
@@ -60,10 +88,6 @@ struct expr_item {
int is_hexbin;
int is_case_sensitive;
int is_valid;
//rule_tag; 只存在schema里
//int have_exdata;
//struct ex_data *ex_data; //hash表
};
struct plugin_item {
@@ -129,14 +153,21 @@ void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* tabl
/* table schema generic API */
struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id);
struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr,
int table_id, enum scan_type type, int *virtual_table_id);
enum table_type table_schema_get_table_type(struct table_schema *table_schema);
int table_schema_get_table_id(struct table_schema *table_schema);
enum scan_type table_schema_get_scan_type(struct table_schema *table_schema);
struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema);
int table_schema_get_valid_flag_column(struct table_schema *table_schema);
/* expr table schema API */
enum scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);
enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);
/* plugin table schema API */
int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
@@ -165,6 +196,8 @@ size_t plugin_table_schema_callback_count(struct table_schema *table_schema);
void plugin_table_schema_all_cb_update(struct table_schema *table_schema, const char *row);
int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns);
#ifdef __cpluscplus
}
#endif

15
src/inc_internal/maat_utils.h
View File

@@ -30,6 +30,8 @@ extern "C"
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#define UNUSED __attribute__((unused))
char *maat_strdup(const char *s);
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len);
@@ -42,6 +44,19 @@ char *str_unescape_and(char *s);
char *str_unescape(char *s);
char *md5_file(const char *filename, char *md5string);
int decrypt_open(const char* file_name, const char* key, const char* algorithm,
unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz);
int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, size_t *out_sz,
const char *key, const char *algorithm, int do_encrypt, char *err_str, size_t err_str_sz);
int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz,
unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz);
size_t memcat(void **dest, size_t offset, size_t *n_dest, const void *src, size_t n_src);
/* system cmd wrapper */
int system_cmd_mkdir(const char* path);