gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add json/redis rule parser

Browse Source
This commit is contained in:
liuwentan
2022-12-03 22:23:41 +08:00
parent 84a271144b
commit ea4c1ba4c3
32 changed files with 6734 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/CMakeLists.txt
View File

@@ -8,8 +8,9 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp
maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp)
set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp
maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp
maat_command.cpp maat_redis_monitor.cpp)
set(LIB_SOURCE_FILES
${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c)
@@ -25,7 +26,7 @@ set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX)
set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe)
set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
target_link_libraries(maat_frame_static adapter-static pthread)
target_link_libraries(maat_frame_static adapter-static hiredis-static pthread crypto z)
# Shared Library Output
#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})

29
src/inc_internal/json2iris.h Normal file
View File

@@ -0,0 +1,29 @@
/*
**********************************************************************************************
* File: json2iris.h
* Description: rule for transform json2iris
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _JSON2IRIS_H_
#define _JSON2IRIS_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "hiredis/hiredis.h"
int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn,
const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx,
char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo);
#ifdef __cpluscplus
}
#endif
#endif

39
src/inc_internal/maat_command.h Normal file
View File

@@ -0,0 +1,39 @@
/*
**********************************************************************************************
* File: maat_command.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_COMMAND_H_
#define _MAAT_COMMAND_H_
#ifdef __cpluscplus
extern "C"
{
#endif
enum maat_operation {
MAAT_OP_DEL = 0,
MAAT_OP_ADD,
MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after
};
struct maat_cmd_line
{
const char *table_name;
const char *table_line;
int rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary.
int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout.
};
int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule);
#ifdef __cpluscplus
}
#endif
#endif

9
src/inc_internal/maat_common.h
View File

@@ -18,6 +18,8 @@ extern "C"
#include <stddef.h>
#include "maat_rule.h"
struct maat_options {
size_t nr_worker_threads;
int rule_effect_interval_ms;
@@ -25,8 +27,11 @@ struct maat_options {
int gc_timeout_ms;
int deferred_load_on;
enum data_source input_mode;
char iris_full_dir[NAME_MAX];
char iris_inc_dir[NAME_MAX];
union {
struct source_iris_ctx iris_ctx;
struct source_json_ctx json_ctx;
struct source_redis_ctx redis_ctx;
};
};
#ifdef __cpluscplus

2
src/inc_internal/maat_config_monitor.h
View File

@@ -28,6 +28,8 @@ void config_monitor_traverse(long long version, const char *idx_dir,
void (*finish_fn)(void *),
void *u_param);
int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz);
#ifdef __cpluscplus
}
#endif

33
src/inc_internal/maat_redis_monitor.h Normal file
View File

@@ -0,0 +1,33 @@
/*
**********************************************************************************************
* File: maat_redis_monitor.h
* Description: maat redis monitor api
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-11-29
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_REDIS_MONITOR_H_
#define _MAAT_REDIS_MONITOR_H_
#ifdef __cpluscplus
extern "C"
{
#endif
#include "maat_rule.h"
#include <stdint.h>
void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx,
void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *),
void *u_param);
#ifdef __cpluscplus
}
#endif
#endif

110
src/inc_internal/maat_rule.h
View File

@@ -21,6 +21,13 @@ extern "C"
#include <limits.h>
#include <sys/time.h>
#include <pthread.h>
#include <sys/queue.h>
#include <openssl/md5.h>
#include "hiredis/hiredis.h"
#include "uthash/uthash.h"
#include "maat_table_schema.h"
#include "maat_command.h"
struct maat_runtime {
/* maat_runtime can be created and destroy dynamic, so need version info */
@@ -37,7 +44,9 @@ struct maat_runtime {
enum data_source {
DATA_SOURCE_NONE = 0,
DATA_SOURCE_IRIS_FILE
DATA_SOURCE_REDIS,
DATA_SOURCE_IRIS_FILE,
DATA_SOURCE_JSON_FILE
};
struct source_iris_ctx {
@@ -45,6 +54,52 @@ struct source_iris_ctx {
char full_dir[NAME_MAX];
};
struct source_json_ctx
{
char json_file[NAME_MAX];
char iris_file[NAME_MAX];
char effective_json_md5[MD5_DIGEST_LENGTH*2+1];
struct timespec last_md5_time;
};
struct source_redis_ctx
{
redisContext *read_ctx;
redisContext *write_ctx;
char redis_ip[64];
uint16_t redis_port;
int redis_db;
time_t last_reconnect_time;
};
struct foreign_key {
int column;
char *key;
size_t key_len;
char *filename;
};
//rm= Redis Maat
struct serial_rule {
enum maat_operation op;//0: delete, 1: add.
unsigned long rule_id;
int label_id;
long long timeout; // absolute unix time.
char table_name[NAME_MAX];
char *table_line;
int n_foreign;
struct foreign_key *f_keys;
TAILQ_ENTRY(serial_rule) entries;
UT_hash_handle hh;
};
#define POSSIBLE_REDIS_REPLY_SIZE 2
struct expected_reply {
int s_rule_seq;
int possible_reply_num;
redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE];
};
struct maat {
char instance_name[NAME_MAX];
@@ -56,6 +111,8 @@ struct maat {
enum data_source input_mode;
union {
struct source_iris_ctx iris_ctx;
struct source_json_ctx json_ctx;
struct source_redis_ctx mr_ctx;
};
int deferred_load;
@@ -72,7 +129,24 @@ struct maat {
int rule_update_checking_interval_ms;
int gc_timeout_ms; //garbage collection timeout_ms;
int cumulative_update_off; //Default: cumulative update on
struct maat_garbage_bin *garbage_bin;
char compile_tn[NAME_MAX];
char group_tn[NAME_MAX];
char group2compile_tn[NAME_MAX];
char group2group_tn[NAME_MAX];
char decrypt_key[NAME_MAX];
char decrypt_algo[NAME_MAX];
int maat_json_is_gzipped;
long long load_specific_version; //Default: Load the Latest. Only valid in redis mode, and maybe failed for too old
char foreign_cont_dir[NAME_MAX];
/* statistics */
long long line_cmd_acc_num;
};
void maat_start_cb(long long new_version, int update_type, void *u_para);
@@ -85,6 +159,40 @@ void *rule_monitor_loop(void *arg);
void maat_read_full_config(struct maat *maat_instance);
/* maat command API for internal */
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db);
redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...);
int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply);
long long maat_cmd_redis_server_time_s(redisContext *c);
long long maat_cmd_read_redis_integer(const redisReply *reply);
int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq);
const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len);
int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time);
void maat_cmd_empty_serial_rule(struct serial_rule *s_rule);
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version,
long long *new_version, struct table_schema_manager* table_schema_mgr,
struct serial_rule **list, int *update_type, int cumulative_off);
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process);
int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num, const char* dir);
void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn);
void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule);
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id,
const char *table_name, const char *line, long long timeout);
#ifdef __cpluscplus
}
#endif

45
src/inc_internal/maat_table_schema.h
View File

@@ -25,13 +25,28 @@ extern "C"
#define MAX_DISTRICT_STR 128
#define MAX_IP_STR 128
#define MAX_KEYWORDS_STR 1024
#define MAX_FOREIGN_CLMN_NUM 8
enum table_type {
TABLE_TYPE_EXPR = 0,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_IP,
TABLE_TYPE_IP_PLUS,
TABLE_TYPE_INTERVAL,
TABLE_TYPE_INTERVAL_PLUS,
TABLE_TYPE_DIGEST,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_FQDN_PLUGIN,
TABLE_TYPE_BOOL_PLUGIN,
//above are physical table
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_COMPOSITION,
TABLE_TYPE_COMPILE,
TABLE_TYPE_GROUP,
TABLE_TYPE_GROUP2GROUP,
TABLE_TYPE_GROUP2COMPILE,
TABLE_TYPE_MAX
};
@@ -42,8 +57,21 @@ enum expr_type {
EXPR_TYPE_MAX
};
enum scan_type {
SCAN_TYPE_INVALID = -1,
SCAN_TYPE_NONE = 0,
SCAN_TYPE_PLUGIN,
SCAN_TYPE_IP_PLUGIN,
SCAN_TYPE_FQDN_PLUGIN,
SCAN_TYPE_BOOL_PLUGIN,
SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING,
SCAN_TYPE_MAX
};
enum match_method {
MATCH_METHOD_SUB=0,
MATCH_METHOD_SUB = 0,
MATCH_METHOD_RIGHT,
MATCH_METHOD_LEFT,
MATCH_METHOD_COMPLETE,
@@ -60,10 +88,6 @@ struct expr_item {
int is_hexbin;
int is_case_sensitive;
int is_valid;
//rule_tag; 只存在schema里
//int have_exdata;
//struct ex_data *ex_data; //hash表
};
struct plugin_item {
@@ -129,14 +153,21 @@ void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* tabl
/* table schema generic API */
struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id);
struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr,
int table_id, enum scan_type type, int *virtual_table_id);
enum table_type table_schema_get_table_type(struct table_schema *table_schema);
int table_schema_get_table_id(struct table_schema *table_schema);
enum scan_type table_schema_get_scan_type(struct table_schema *table_schema);
struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema);
int table_schema_get_valid_flag_column(struct table_schema *table_schema);
/* expr table schema API */
enum scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);
enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);
/* plugin table schema API */
int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
@@ -165,6 +196,8 @@ size_t plugin_table_schema_callback_count(struct table_schema *table_schema);
void plugin_table_schema_all_cb_update(struct table_schema *table_schema, const char *row);
int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns);
#ifdef __cpluscplus
}
#endif

15
src/inc_internal/maat_utils.h
View File

@@ -30,6 +30,8 @@ extern "C"
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#define UNUSED __attribute__((unused))
char *maat_strdup(const char *s);
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len);
@@ -42,6 +44,19 @@ char *str_unescape_and(char *s);
char *str_unescape(char *s);
char *md5_file(const char *filename, char *md5string);
int decrypt_open(const char* file_name, const char* key, const char* algorithm,
unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz);
int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, size_t *out_sz,
const char *key, const char *algorithm, int do_encrypt, char *err_str, size_t err_str_sz);
int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz,
unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz);
size_t memcat(void **dest, size_t offset, size_t *n_dest, const void *src, size_t n_src);
/* system cmd wrapper */
int system_cmd_mkdir(const char* path);

1207
src/json2iris.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

92
src/maat_api.cpp
View File

@@ -12,13 +12,16 @@
#include <string.h>
#include "utils.h"
#include "json2iris.h"
#include "maat/maat.h"
#include "maat_rule.h"
#include "maat_common.h"
#include "maat_kv.h"
#include "maat_command.h"
#include "maat_table_schema.h"
#include "maat_table_runtime.h"
#include "maat_config_monitor.h"
#include "maat_redis_monitor.h"
struct maat_options* maat_options_new(void)
{
@@ -65,6 +68,7 @@ int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms)
int maat_options_set_deferred_load_on(struct maat_options *opts)
{
opts->deferred_load_on = 1;
return 0;
}
@@ -74,8 +78,9 @@ int maat_options_set_iris_full_dir(struct maat_options *opts, const char *full_d
return -1;
}
memcpy(opts->iris_full_dir, full_dir, strlen(full_dir));
memcpy(opts->iris_ctx.full_dir, full_dir, strlen(full_dir));
opts->input_mode = DATA_SOURCE_IRIS_FILE;
return 0;
}
@@ -85,26 +90,91 @@ int maat_options_set_iris_inc_dir(struct maat_options *opts, const char *inc_dir
return -1;
}
memcpy(opts->iris_inc_dir, inc_dir, strlen(inc_dir));
memcpy(opts->iris_ctx.inc_dir, inc_dir, strlen(inc_dir));
opts->input_mode = DATA_SOURCE_IRIS_FILE;
return 0;
}
int maat_options_set_json_file(struct maat_options *opts, const char *json_filename)
{
strncpy(opts->json_ctx.json_file, json_filename, sizeof(opts->json_ctx.json_file));
opts->input_mode = DATA_SOURCE_JSON_FILE;
return 0;
}
int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip)
{
memcpy(opts->redis_ctx.redis_ip, redis_ip, strlen(redis_ip));
opts->input_mode = DATA_SOURCE_REDIS;
return 0;
}
int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port)
{
opts->redis_ctx.redis_port = redis_port;
return 0;
}
int maat_options_set_redis_db_index(struct maat_options *opts, int db_index)
{
opts->redis_ctx.redis_db = db_index;
return 0;
}
void maat_read_full_config(struct maat *maat_instance)
{
int ret = -1;
char err_str[NAME_MAX] = {0};
struct source_redis_ctx *mr_ctx = NULL;
switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS:
mr_ctx = &(maat_instance->mr_ctx);
fprintf(stdout, "Maat initiate from Redis %s:%hu db%d.",
mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
if (mr_ctx->read_ctx != NULL) {
redis_monitor_traverse(maat_instance->maat_version, mr_ctx,
maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance);
}
if (NULL == maat_instance->creating_maat_rt) {
fprintf(stderr, "At initiation: NO effective rule in %s.",
maat_instance->iris_ctx.full_dir);
}
break;
case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version,
maat_instance->iris_ctx.full_dir,
maat_start_cb,
maat_update_cb,
maat_finish_cb,
maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance);
if (NULL == maat_instance->creating_maat_rt) {
fprintf(stderr, "At initiation: NO effective rule in %s.",
maat_instance->iris_ctx.full_dir);
}
break;
case DATA_SOURCE_JSON_FILE:
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str));
if (ret < 0) {
fprintf(stderr, "Maat re-initiate with JSON file %s failed: %s", maat_instance->json_ctx.json_file, err_str);
return;
}
config_monitor_traverse(maat_instance->maat_version,
maat_instance->json_ctx.iris_file,
maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance);
if (NULL == maat_instance->creating_maat_rt) {
fprintf(stderr, "At initiation: NO effective rule in %s.",
maat_instance->json_ctx.iris_file);
}
break;
default:
break;
}
@@ -134,9 +204,17 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
maat_instance->input_mode = opts->input_mode;
switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS:
memcpy(maat_instance->mr_ctx.redis_ip, opts->redis_ctx.redis_ip, strlen(opts->redis_ctx.redis_ip));
maat_instance->mr_ctx.redis_port = opts->redis_ctx.redis_port;
maat_instance->mr_ctx.redis_db = opts->redis_ctx.redis_db;
break;
case DATA_SOURCE_IRIS_FILE:
memcpy(maat_instance->iris_ctx.full_dir, opts->iris_full_dir, strlen(opts->iris_full_dir));
memcpy(maat_instance->iris_ctx.inc_dir, opts->iris_inc_dir, strlen(opts->iris_inc_dir));
memcpy(maat_instance->iris_ctx.full_dir, opts->iris_ctx.full_dir, strlen(opts->iris_ctx.full_dir));
memcpy(maat_instance->iris_ctx.inc_dir, opts->iris_ctx.inc_dir, strlen(opts->iris_ctx.inc_dir));
break;
case DATA_SOURCE_JSON_FILE:
memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file));
break;
default:
fprintf(stderr, "data source unsupported:%d\n", maat_instance->input_mode);

348
src/maat_command.cpp Normal file
View File

@@ -0,0 +1,348 @@
/*
**********************************************************************************************
* File: maat_command.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include <stdlib.h>
#include <assert.h>
#include <unistd.h>
#include <string.h>
#include "utils.h"
#include "maat_utils.h"
#include "maat_command.h"
#include "maat_rule.h"
#include "hiredis/hiredis.h"
#include "maat_config_monitor.h"
#include "maat_table_schema.h"
extern const char *foreign_source_prefix;
extern const char *mr_key_prefix;
extern const char *mr_expire_lock;
extern const long mr_expire_lock_time;
extern const char *mr_status_sset;
extern const char *mr_version_sset;
extern const char *mr_label_sset;
redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...)
{
va_list ap;
void *reply = NULL;
int ret = REDIS_ERR;
int retry = 0;
while (reply == NULL && retry < 2 && ret != REDIS_OK) {
va_start(ap,format);
reply = redisvCommand(c,format,ap);
va_end(ap);
if (NULL == reply) {
ret = redisReconnect(c);
retry++;
}
}
return (redisReply *)reply;
}
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db)
{
struct timeval connect_timeout;
connect_timeout.tv_sec = 0;
connect_timeout.tv_usec = 100 * 1000; // 100 ms
redisContext *c = redisConnectWithTimeout(redis_ip, redis_port, connect_timeout);
if (NULL == c || c->err) {
fprintf(stderr, "Unable to connect redis server %s:%d db%d, error: %s",
redis_ip, redis_port, redis_db, c == NULL ? "Unknown" : c->errstr);
if (c != NULL) {
redisFree(c);
}
return NULL;
}
redisEnableKeepAlive(c);
redisReply *reply = maat_cmd_wrap_redis_command(c, "select %d", redis_db);
freeReplyObject(reply);
reply = NULL;
return c;
}
struct s_rule_array
{
int cnt;
int size;
struct serial_rule *array;
};
void save_serial_rule(void *data, void *user)
{
struct s_rule_array *array = (struct s_rule_array *)user;
int i = array->cnt;
memcpy(&(array->array[i]), data, sizeof(struct serial_rule));
array->array[i].op = MAAT_OP_ADD;
}
void maat_cmd_empty_serial_rule(struct serial_rule *s_rule)
{
if (s_rule->table_line != NULL) {
FREE(s_rule->table_line);
}
if (s_rule->n_foreign > 0) {
for (int i = 0; i < s_rule->n_foreign; i++) {
FREE(s_rule->f_keys[i].filename);
FREE(s_rule->f_keys[i].key);
}
FREE(s_rule->f_keys);
}
memset(s_rule, 0, sizeof(struct serial_rule));
}
int connect_redis_for_write(struct source_redis_ctx *mr_ctx)
{
assert(mr_ctx->write_ctx == NULL);
mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
if (NULL == mr_ctx->write_ctx) {
return -1;
} else {
return 0;
}
}
redisContext *get_redis_ctx_for_write(struct maat *maat_instance)
{
if (NULL == maat_instance->mr_ctx.write_ctx) {
int ret = connect_redis_for_write(&(maat_instance->mr_ctx));
if(ret!=0)
{
return NULL;
}
}
return maat_instance->mr_ctx.write_ctx;
}
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id,
const char *table_name, const char *line, long long timeout)
{
memset(rule, 0, sizeof(struct serial_rule));
rule->op = op;
rule->rule_id = rule_id;
rule->timeout = timeout;
assert(strlen(table_name) < sizeof(rule->table_name));
strncpy(rule->table_name, table_name, sizeof(rule->table_name));
if (line != NULL) {
rule->table_line = maat_strdup(line);
}
}
int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq)
{
int column_seq = 0;
switch (table_type) {
case TABLE_TYPE_EXPR:
column_seq = 7;
break;
case TABLE_TYPE_IP:
column_seq = 14;
break;
case TABLE_TYPE_IP_PLUS:
column_seq = 18;
break;
case TABLE_TYPE_COMPILE:
column_seq = 8;
break;
case TABLE_TYPE_PLUGIN:
case TABLE_TYPE_IP_PLUGIN:
case TABLE_TYPE_FQDN_PLUGIN:
case TABLE_TYPE_BOOL_PLUGIN:
if (valid_column_seq < 0) {
return -1;
}
column_seq = valid_column_seq;
break;
case TABLE_TYPE_INTERVAL:
column_seq = 5;
break;
case TABLE_TYPE_INTERVAL_PLUS:
column_seq = 6;
break;
case TABLE_TYPE_DIGEST:
column_seq = 6;
break;
case TABLE_TYPE_SIMILARITY:
column_seq = 5;
break;
case TABLE_TYPE_EXPR_PLUS:
column_seq = 8;
break;
case TABLE_TYPE_GROUP2COMPILE:
case TABLE_TYPE_GROUP2GROUP:
column_seq = 3;
break;
default:
assert(0);
}
size_t offset = 0;
size_t len = 0;
int ret = get_column_pos(line, column_seq, &offset, &len);
// 0 is also a valid value for some non-MAAT producer.
if (ret < 0 || offset >= strlen(line) || (line[offset] != '1' && line[offset] != '0')) {
return -1;
}
return offset;
}
long long maat_cmd_redis_server_time_s(redisContext *c)
{
long long server_time = 0;
redisReply *data_reply = maat_cmd_wrap_redis_command(c, "TIME");
if (data_reply->type == REDIS_REPLY_ARRAY) {
server_time = atoll(data_reply->element[0]->str);
freeReplyObject(data_reply);
data_reply = NULL;
}
return server_time;
}
const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len)
{
size_t i = 0;
int j = 0;
size_t start=0, end=0;
size_t line_len = strlen(line);
for (i = 0; i < line_len; i++) {
if (line[i] != ' ' && line[i] != '\t') {
continue;
}
j++;
if (j == Nth - 1) {
start = i + 1;
}
if(j == Nth) {
end = i;
break;
}
}
if (start == end) {
return NULL;
}
if (end == 0) {
end = i;
}
*column_len = end - start;
return line + start;
}
long long maat_cmd_read_redis_integer(const redisReply *reply)
{
switch (reply->type) {
case REDIS_REPLY_INTEGER:
return reply->integer;
break;
case REDIS_REPLY_ARRAY:
assert(reply->element[0]->type == REDIS_REPLY_INTEGER);
return reply->element[0]->integer;
break;
case REDIS_REPLY_STRING:
return atoll(reply->str);
break;
default:
return -1;
break;
}
return 0;
}
int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply)
{
return redisGetReply(c, (void **)reply);
}
int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule)
{
int i = 0;
int ret = 0;
long long absolute_expire_time = 0;
redisContext *write_ctx = get_redis_ctx_for_write(maat_instance);
if (NULL == write_ctx) {
return -1;
}
long long server_time = maat_cmd_redis_server_time_s(write_ctx);
if(!server_time) {
return -1;
}
struct serial_rule *s_rule = ALLOC(struct serial_rule, 1);
int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, line_rule->table_name);
if (table_id < 0) {
fprintf(stderr, "Command set line id %d failed: unknown table %s.", line_rule->rule_id, line_rule->table_name);
FREE(s_rule);
return -1;
}
struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
if (!table_schema) {
FREE(s_rule);
return -1;
}
int valid_flag_column = table_schema_get_valid_flag_column(table_schema);
if (valid_flag_column < 0) {
fprintf(stderr, "Command set line id %d failed: table %s is not a plugin or ip_plugin table.",
line_rule->rule_id, line_rule->table_name);
FREE(s_rule);
return -1;
}
enum table_type table_type = table_schema_get_table_type(table_schema);
int is_valid = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_flag_column);
if (line_rule->expire_after > 0) {
absolute_expire_time = server_time + line_rule->expire_after;
}
maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name,
line_rule->table_line, absolute_expire_time);
int success_cnt = maat_cmd_exec_serial_rule(write_ctx, s_rule, 1, server_time);
if (success_cnt != 1) {
ret = -1;
goto error_out;
}
ret = success_cnt;
maat_instance->line_cmd_acc_num += success_cnt;
error_out:
maat_cmd_empty_serial_rule(s_rule);
FREE(s_rule);
return ret;
}

84
src/maat_config_monitor.cpp
View File

@@ -14,10 +14,13 @@
#include <limits.h>
#include <sys/stat.h>
#include <dirent.h>
#include <unistd.h>
#include "maat_config_monitor.h"
#include "maat_utils.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_rule.h"
#include "json2iris.h"
#include "maat_config_monitor.h"
#define CM_MAX_TABLE_NUM 256
#define MAX_CONFIG_LINE (1024 * 16)
@@ -345,4 +348,81 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
}
free(idx_path_array);
}
int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz)
{
int ret = 0;
struct stat fstat_buf;
unsigned char *json_buff = NULL;
unsigned char *decrypted_buff = NULL;
unsigned char *uncompressed_buff = NULL;
size_t json_buff_sz = 0;
size_t decrypted_buff_sz = 0;
size_t uncompressed_buff_sz = 0;
fprintf(stdout, "Maat initial with JSON file %s, formating..", json_filename);
if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) {
ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo,
(unsigned char **)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz);
if (ret < 0) {
fprintf(stderr, "Decrypt Maat JSON file %s failed.", json_filename);
return -1;
}
json_buff=decrypted_buff;
json_buff_sz=decrypted_buff_sz;
}
if (maat_instance->maat_json_is_gzipped) {
ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz);
free(json_buff);
if (ret < 0) {
fprintf(stderr, "Uncompress Maat JSON file %s failed.", json_filename);
return -1;
}
json_buff = uncompressed_buff;
json_buff_sz = uncompressed_buff_sz;
}
//decryption failed or no decryption
if (NULL == json_buff) {
ret = load_file_to_memory(json_filename, &json_buff, &json_buff_sz);
if (ret < 0) {
fprintf(stderr, "Read Maat JSON file %s failed.", json_filename);
return -1;
}
}
ret = json2iris((const char*)json_buff, json_filename,
maat_instance->compile_tn, maat_instance->group2compile_tn, maat_instance->group2group_tn,
NULL,
maat_instance->json_ctx.iris_file,
sizeof(maat_instance->json_ctx.iris_file),
strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL,
strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL);
free(json_buff);
json_buff = NULL;
if (ret < 0) {
return -1;
}
if (!maat_instance->is_running) {
strncpy(maat_instance->json_ctx.json_file, json_filename, sizeof(maat_instance->json_ctx.json_file));
}
ret=stat(json_filename, &fstat_buf);
maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim;
md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5);
fprintf(stdout, "JSON file %s md5: %s, generate index file %s OK.",
maat_instance->json_ctx.json_file,
maat_instance->json_ctx.effective_json_md5,
maat_instance->json_ctx.iris_file);
maat_instance->input_mode = DATA_SOURCE_JSON_FILE;
return 0;
}

1403
src/maat_redis_monitor.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

66
src/maat_rule.cpp
View File

@@ -14,12 +14,16 @@
#include <limits.h>
#include <string.h>
#include <sys/prctl.h>
#include <sys/stat.h>
#include <assert.h>
#include "utils.h"
#include "json2iris.h"
#include "maat_utils.h"
#include "maat_rule.h"
#include "maat_config_monitor.h"
#include "utils.h"
#include "maat_utils.h"
#include "maat_redis_monitor.h"
#include "maat_table_runtime.h"
#include "maat_table_schema.h"
@@ -100,7 +104,16 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param)
struct maat *maat_instance =(struct maat *)u_param;
struct maat_runtime* maat_rt = NULL;
int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, table_name);
if (table_id < 0) {
fprintf(stderr, "update warning, unknown table name %s\n", table_name);
return -1;
}
struct table_schema* table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
if (NULL == table_schema) {
fprintf(stderr, "update warning, table name %s doesn't have table schema\n", table_name);
return -1;
}
if (maat_instance->creating_maat_rt != NULL) {
maat_rt = maat_instance->creating_maat_rt;
@@ -177,10 +190,21 @@ void *rule_monitor_loop(void *arg)
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
char md5_tmp[MD5_DIGEST_LENGTH * 2 + 1] = {0};
char err_str[NAME_MAX] = {0};
struct stat attrib;
while (maat_instance->is_running) {
usleep(maat_instance->rule_update_checking_interval_ms * 1000);
if( 0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS:
redis_monitor_traverse(maat_instance->maat_version,
&(maat_instance->mr_ctx),
maat_start_cb,
maat_update_cb,
maat_finish_cb,
maat_instance);
break;
case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version,
maat_instance->iris_ctx.inc_dir,
@@ -189,6 +213,29 @@ void *rule_monitor_loop(void *arg)
maat_finish_cb,
maat_instance);
break;
case DATA_SOURCE_JSON_FILE:
memset(md5_tmp, 0, sizeof(md5_tmp));
stat(maat_instance->json_ctx.json_file, &attrib);
if (memcmp(&attrib.st_ctim, &(maat_instance->json_ctx.last_md5_time), sizeof(attrib.st_ctim))) {
maat_instance->json_ctx.last_md5_time = attrib.st_ctim;
md5_file(maat_instance->json_ctx.json_file, md5_tmp);
if (0 != strcmp(md5_tmp, maat_instance->json_ctx.effective_json_md5)) {
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str));
if (ret < 0) {
fprintf(stdout, "Maat re-initiate with JSON file %s (md5=%s)failed: %s",
maat_instance->json_ctx.json_file, md5_tmp, err_str);
} else {
config_monitor_traverse(0, maat_instance->json_ctx.iris_file,
maat_start_cb,
maat_update_cb,
maat_finish_cb,
maat_instance);
fprintf(stdout, "Maat re-initiate with JSON file %s success, md5: %s",
maat_instance->json_ctx.json_file, md5_tmp);
}
}
}
break;
default:
break;
}
@@ -231,7 +278,20 @@ void *rule_monitor_loop(void *arg)
maat_runtime_destroy(maat_instance->maat_rt);
maat_garbage_bin_free(maat_instance->garbage_bin);
table_schema_manager_destroy(maat_instance->table_schema_mgr);
free(maat_instance);
if (maat_instance->input_mode == DATA_SOURCE_REDIS) {
if (maat_instance->mr_ctx.read_ctx != NULL) {
redisFree(maat_instance->mr_ctx.read_ctx);
maat_instance->mr_ctx.read_ctx = NULL;
}
if (maat_instance->mr_ctx.write_ctx != NULL) {
redisFree(maat_instance->mr_ctx.write_ctx);
maat_instance->mr_ctx.write_ctx = NULL;
}
}
FREE(maat_instance);
return NULL;
}

2
src/maat_table_runtime.cpp
View File

@@ -36,7 +36,7 @@ struct plugin_user_ctx {
};
struct expr_runtime {
enum scan_mode scan_mode;
enum hs_scan_mode scan_mode;
struct adapter_hs *hs;
struct adapter_hs_stream *hs_stream;
struct rcu_hash_table *htable;

215
src/maat_table_schema.cpp
View File

@@ -33,7 +33,7 @@ struct expr_table_schema {
int match_method_column;
int is_hexbin_column;
int is_valid_column; /* valid means add, invalid means delete */
enum scan_mode scan_mode; /* adapter_hs scan mode */
enum hs_scan_mode scan_mode; /* adapter_hs scan mode */
};
#define MAX_PLUGIN_PER_TABLE 32
@@ -60,6 +60,10 @@ struct ip_plugin_table_schema {
struct ex_data_schema ex_schema;
};
struct virtual_table_schema {
int physical_table_id[SCAN_TYPE_MAX];
};
struct table_schema {
int table_id;
char table_name[NAME_MAX];
@@ -68,6 +72,7 @@ struct table_schema {
struct expr_table_schema expr;
struct plugin_table_schema plugin;
struct ip_plugin_table_schema ip_plugin;
struct virtual_table_schema virtual_table;
};
};
@@ -388,6 +393,10 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_
unsigned char *json_buff = NULL;
size_t json_buff_sz = 0;
if (NULL == table_info_path) {
return NULL;
}
int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz);
if (ret < 0) {
fprintf(stderr, "Maat read table info %s error.\n", table_info_path);
@@ -417,8 +426,8 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_
maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
maat_kv_register(reserved_word_map, "block", SCAN_MODE_BLOCK);
maat_kv_register(reserved_word_map, "stream", SCAN_MODE_STREAM);
maat_kv_register(reserved_word_map, "block", HS_SCAN_MODE_BLOCK);
maat_kv_register(reserved_word_map, "stream", HS_SCAN_MODE_STREAM);
struct table_schema_manager *table_schema_mgr = ALLOC(struct table_schema_manager, 1);
struct table_schema **pptable = table_schema_mgr->schema_table;
@@ -471,6 +480,10 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_
void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr)
{
if (NULL == table_schema_mgr) {
return;
}
for (size_t i = 0; i < MAX_TABLE_NUM; i++) {
if (NULL == table_schema_mgr->schema_table[i]) {
continue;
@@ -480,11 +493,15 @@ void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr)
}
maat_kv_store_free(table_schema_mgr->tablename2id_map);
free(table_schema_mgr);
FREE(table_schema_mgr);
}
int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name)
{
if (NULL == table_schema_mgr || NULL == table_name) {
return -1;
}
int table_id = -1;
int ret = maat_kv_read(table_schema_mgr->tablename2id_map, table_name, &table_id);
@@ -497,6 +514,10 @@ int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_
enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int id)
{
if (NULL == table_schema_mgr) {
return TABLE_TYPE_MAX;
}
if (table_schema_mgr->schema_table[id] == NULL) {
return TABLE_TYPE_MAX;
}
@@ -511,6 +532,10 @@ size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_m
void table_schema_manager_all_plugin_cb_start(struct table_schema_manager *table_schema_mgr, int update_type)
{
if (NULL == table_schema_mgr) {
return;
}
struct table_schema *ptable = NULL;
struct plugin_table_schema *plugin_schema = NULL;
@@ -536,6 +561,10 @@ void table_schema_manager_all_plugin_cb_start(struct table_schema_manager *table
void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* table_schema_mgr)
{
if (NULL == table_schema_mgr) {
return;
}
struct table_schema *ptable = NULL;
struct plugin_table_schema *plugin_schema = NULL;
@@ -606,19 +635,149 @@ enum match_method int_to_match_method_type(int match_method_type)
struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id)
{
if ((NULL == table_schema_mgr) || (table_id < 0)) {
return NULL;
}
return table_schema_mgr->schema_table[table_id];
}
struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr,
int table_id, enum scan_type scan_type, int *virtual_table_id)
{
enum scan_type table_scan_type;
struct table_schema **pptable = table_schema_mgr->schema_table;
size_t n_table = MAX_TABLE_NUM;
if ((unsigned int)table_id > n_table) {
return NULL;
}
if (NULL == pptable[table_id]) {
return NULL;
}
struct table_schema *ptable = pptable[table_id];
if (NULL == ptable) {
return NULL;
}
struct table_schema *p_physical_table = NULL;
if (ptable->table_type == TABLE_TYPE_VIRTUAL) {
p_physical_table = pptable[ptable->virtual_table.physical_table_id[scan_type]];
*virtual_table_id = table_id;
} else {
p_physical_table = ptable;
if(virtual_table_id) {
*virtual_table_id = 0;
}
}
table_scan_type = table_schema_get_scan_type(p_physical_table);
if (table_scan_type != scan_type) {
return NULL;
}
return p_physical_table;
}
enum table_type table_schema_get_table_type(struct table_schema *table_schema)
{
if (NULL == table_schema) {
return TABLE_TYPE_MAX;
}
return table_schema->table_type;
}
int table_schema_get_table_id(struct table_schema *table_schema)
{
if (NULL == table_schema) {
return -1;
}
return table_schema->table_id;
}
enum scan_type table_schema_get_scan_type(struct table_schema *table_schema)
{
enum scan_type ret = SCAN_TYPE_INVALID;
if (NULL == table_schema) {
return ret;
}
switch (table_schema->table_type) {
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
case TABLE_TYPE_SIMILARITY:
case TABLE_TYPE_DIGEST:
ret = SCAN_TYPE_STRING;
break;
case TABLE_TYPE_INTERVAL:
case TABLE_TYPE_INTERVAL_PLUS:
ret = SCAN_TYPE_INTERVAL;
break;
case TABLE_TYPE_IP:
case TABLE_TYPE_IP_PLUS:
case TABLE_TYPE_COMPOSITION:
ret = SCAN_TYPE_IP;
break;
case TABLE_TYPE_PLUGIN:
ret = SCAN_TYPE_PLUGIN;
break;
case TABLE_TYPE_IP_PLUGIN:
ret = SCAN_TYPE_IP;
break;
case TABLE_TYPE_FQDN_PLUGIN:
ret = SCAN_TYPE_FQDN_PLUGIN;
break;
case TABLE_TYPE_BOOL_PLUGIN:
ret = SCAN_TYPE_BOOL_PLUGIN;
break;
case TABLE_TYPE_COMPILE:
ret = SCAN_TYPE_NONE;
break;
default:
break;
}
return ret;
}
int table_schema_get_valid_flag_column(struct table_schema *table_schema)
{
int valid_flag_column = -1;
if (NULL == table_schema) {
return valid_flag_column;
}
switch (table_schema->table_type) {
case TABLE_TYPE_EXPR:
valid_flag_column = table_schema->expr.is_valid_column;
break;
case TABLE_TYPE_PLUGIN:
valid_flag_column = table_schema->plugin.is_valid_column;
break;
case TABLE_TYPE_IP_PLUGIN:
valid_flag_column = table_schema->ip_plugin.is_valid_column;
break;
/*
case TABLE_TYPE_FQDN_PLUGIN:
valid_flag_column = table_schema->fqdn_plugin.valid_flag_column;
break;
case TABLE_TYPE_BOOL_PLUGIN:
valid_flag_column = table_schema->bool_plugin.valid_flag_column;
break;*/
default:
valid_flag_column = -1;
break;
}
return valid_flag_column;
}
int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema, struct expr_item *expr_item)
{
size_t column_offset = 0;
@@ -778,6 +937,10 @@ int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schem
struct table_item *
table_schema_line_to_item(const char *line, struct table_schema *table_schema)
{
if (NULL == line || NULL == table_schema) {
return NULL;
}
int ret = -1;
struct table_item *table_item = ALLOC(struct table_item, 1);
@@ -812,10 +975,14 @@ error:
return NULL;
}
enum scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema)
enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema)
{
if (NULL == table_schema) {
return HS_SCAN_MODE_MAX;
}
if (table_schema->table_type != TABLE_TYPE_EXPR) {
return SCAN_MODE_MAX;
return HS_SCAN_MODE_MAX;
}
return table_schema->expr.scan_mode;
@@ -827,7 +994,7 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp)
{
if (NULL == new_func || NULL == free_func || NULL == dup_func) {
if (NULL == table_schema || NULL == new_func || NULL == free_func || NULL == dup_func) {
assert(0);
fprintf(stderr, "%s failed: invalid parameter", __FUNCTION__);
return -1;
@@ -857,6 +1024,10 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schema *table_schema)
{
if (NULL == table_schema) {
return NULL;
}
struct ex_data_schema *ex_schema = NULL;
switch (table_schema->table_type) {
@@ -875,6 +1046,10 @@ struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schem
int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema)
{
if (NULL == table_schema) {
return -1;
}
struct ex_data_schema *ex_schema = NULL;
switch (table_schema->table_type) {
@@ -897,6 +1072,10 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m
maat_finish_callback_t *finish,
void *u_para)
{
if ((NULL == table_schema_mgr) || (table_id < 0)) {
return -1;
}
struct table_schema *ptable = table_schema_get(table_schema_mgr, table_id);
if (NULL == ptable) {
fprintf(stderr, "table_id:%d unregistered, can't add callback func", table_id);
@@ -945,6 +1124,10 @@ size_t plugin_table_schema_callback_count(struct table_schema *table_schema)
void plugin_table_schema_all_cb_update(struct table_schema* table_schema, const char *row)
{
if (NULL == table_schema || NULL == row) {
return;
}
struct plugin_table_schema *plugin_schema = NULL;
switch (table_schema->table_type) {
@@ -963,4 +1146,22 @@ void plugin_table_schema_all_cb_update(struct table_schema* table_schema, const
default:
break;
}
}
int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns)
{
if (NULL == table_schema) {
return -1;
}
if (table_schema->table_type != TABLE_TYPE_PLUGIN) {
return 0;
}
int n_foreign = table_schema->plugin.n_foreign;
for (int i = 0; i < n_foreign; i++) {
foreign_columns[i] = table_schema->plugin.foreign_columns[i];
}
return n_foreign;
}

195
src/maat_utils.cpp
View File

@@ -11,6 +11,9 @@
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <zlib.h>
#include <openssl/md5.h>
#include <openssl/evp.h>
#include "maat_utils.h"
@@ -19,6 +22,7 @@ char *maat_strdup(const char *s)
if (NULL == s) {
return NULL;
}
char *d = (char *)malloc(strlen(s) + 1);
memcpy(d, s, strlen(s) + 1);
@@ -27,10 +31,14 @@ char *maat_strdup(const char *s)
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len)
{
const char *seps=" \t";
char *saveptr=NULL, *subtoken=NULL, *str=NULL;
int i = 0;
int ret = -1;
char *str = NULL;
char *saveptr = NULL;
char *subtoken = NULL;
const char *seps = " \t";
char *dup_line = maat_strdup(line);
int i = 0, ret = -1;
for (str = dup_line; ; str = NULL) {
subtoken = strtok_r(str, seps, &saveptr);
if (subtoken == NULL)
@@ -53,7 +61,7 @@ int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *o
int ret = 0;
FILE *fp = NULL;
struct stat fstat_buf;
size_t read_size=0;
size_t read_size = 0;
ret = stat(file_name, &fstat_buf);
if (ret != 0) {
@@ -186,4 +194,181 @@ int system_cmd_mkdir(const char *path)
char cmd[MAX_SYSTEM_CMD_LEN] = {0};
snprintf(cmd, sizeof(cmd), "mkdir -p %s", path);
return system(cmd);
}
}
char *md5_file(const char *filename, char *md5string)
{
unsigned char md5[MD5_DIGEST_LENGTH] = {0};
struct stat file_info;
stat(filename, &file_info);
size_t file_size = file_info.st_size;
FILE *fp = fopen(filename,"r");
if (NULL == fp) {
return NULL;
}
char *file_buff = (char *)malloc(file_size);
fread(file_buff, 1, file_size, fp);
fclose(fp);
MD5((const unsigned char *)(file_buff), (unsigned long)(file_size), md5);
for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
sprintf(&md5string[i*2], "%02x", (unsigned int)md5[i]);
}
free(file_buff);
return md5string;
}
int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, size_t *out_sz,
const char *key, const char *algorithm, int do_encrypt,
char *err_str, size_t err_str_sz)
{
OpenSSL_add_all_algorithms();
const EVP_CIPHER *cipher = EVP_get_cipherbyname(algorithm);
if (NULL == cipher) {
snprintf(err_str, err_str_sz, "Cipher %s is not supported.", algorithm);
return 0;
}
const EVP_MD *dgst = EVP_get_digestbyname("md5");
if (NULL == dgst) {
snprintf(err_str, err_str_sz, "Get MD5 object failed.");
return 0;
}
const unsigned char *salt = NULL;
unsigned char cipher_key[EVP_MAX_KEY_LENGTH];
unsigned char cipher_iv[EVP_MAX_IV_LENGTH];
memset(cipher_key,0,sizeof(cipher_key));
memset(cipher_iv,0,sizeof(cipher_iv));
int ret = EVP_BytesToKey(cipher, dgst, salt, (unsigned char *)key,
strlen((const char *)key), 1, cipher_key, cipher_iv);
if(0 == ret) {
snprintf(err_str, err_str_sz, "Key and IV generatioin failed.");
return 0;
}
/* Don't set key or IV right away; we want to check lengths */
EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, do_encrypt);
OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16 == 0);
OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16);
/* Now we can set key and IV */
//It should be set to 1 for encryption, 0 for decryption and -1 to leave the value unchanged (the actual value of 'enc' being supplied in a previous call).
EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, -1);
int out_blk_len = 0;
int out_buff_offset = 0;
int out_buff_len = inlen + EVP_CIPHER_block_size(cipher) - 1;
*pp_out = (unsigned char *)malloc(out_buff_len * sizeof(unsigned char));
if (!EVP_CipherUpdate(ctx, *pp_out + out_buff_offset, &out_blk_len, inbuf, inlen)) {
snprintf(err_str, err_str_sz, "EVP_CipherUpdate failed.");
EVP_CIPHER_CTX_free(ctx);
goto error_out;
}
out_buff_offset += out_blk_len;
if (!EVP_CipherFinal_ex(ctx, *pp_out+out_buff_offset, &out_blk_len)) {
snprintf(err_str, err_str_sz, "EVP_CipherFinal_ex failed. Maybe password is wrong?");
EVP_CIPHER_CTX_free(ctx);
goto error_out;
}
out_buff_offset += out_blk_len;
EVP_CIPHER_CTX_free(ctx);
*out_sz = out_buff_offset;
return 0;
error_out:
free(*pp_out);
*pp_out = NULL;
return -1;
}
int decrypt_open(const char* file_name, const char* key, const char* algorithm,
unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz)
{
size_t file_sz = 0;
unsigned char *file_buff = NULL;
int ret = load_file_to_memory(file_name, &file_buff, &file_sz);
if (ret < 0) {
return -1;
}
ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz);
free(file_buff);
file_buff = NULL;
return ret;
}
int gzip_uncompress_one_try(const unsigned char *in_compressed_data, size_t in_compressed_sz,
unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz)
{
z_stream strm;
strm.zalloc = NULL;
strm.zfree = NULL;
strm.opaque = NULL;
strm.avail_in = in_compressed_sz;
strm.avail_out = *out_uncompressed_sz;
strm.next_in = (Bytef *) in_compressed_data;
strm.next_out = *out_uncompressed_data;
int ret = -1;
ret = inflateInit2(&strm, MAX_WBITS+16);
if (ret == Z_OK) {
ret = inflate(&strm, Z_FINISH);
if (ret == Z_STREAM_END) {
*out_uncompressed_sz = strm.total_out;
ret = inflateEnd(&strm);
return ret;
}
}
inflateEnd(&strm);
return ret;
}
int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz,
unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz)
{
int z_result;
int ret = -1;
size_t buffer_sz = in_compressed_sz * 2;
*out_uncompressed_data = (unsigned char *)malloc(buffer_sz);
do {
*out_uncompressed_sz=buffer_sz;
z_result = gzip_uncompress_one_try(in_compressed_data, in_compressed_sz,
out_uncompressed_data, out_uncompressed_sz);
switch (z_result) {
case Z_OK:
ret = 0;
break;
case Z_BUF_ERROR:
buffer_sz *= 2;
*out_uncompressed_data = (unsigned char *)realloc(*out_uncompressed_data, buffer_sz);
break;
default:
ret = -1;
break;
}
} while (z_result == Z_BUF_ERROR);
return ret;
}
size_t memcat(void **dest, size_t offset, size_t *n_dest, const void *src, size_t n_src)
{
if (*n_dest < offset + n_src) {
*n_dest = (offset + n_src) * 2;
*dest = realloc(*dest, sizeof(char) * (*n_dest));
}
memcpy((char *) * dest + offset, src, n_src);
return n_src;
}