gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

maat_redis_tool支持dump内容外键。

Browse Source
This commit is contained in:
zhengchao
2018-09-26 12:09:08 +08:00
parent ad24e175dc
commit 4d5b5f1f45
6 changed files with 120 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/CMakeLists.txt
View File

@@ -1,9 +1,9 @@
cmake_minimum_required(VERSION 2.6)
cmake_minimum_required(VERSION 3.5)
project(maatframe)
set(MAAT_FRAME_MAJOR_VERSION 2)
set(MAAT_FRAME_MINOR_VERSION 2)
set(MAAT_FRAME_PATCH_VERSION 20180923)
set(MAAT_FRAME_PATCH_VERSION 20180926)
set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION})
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")

126
src/entry/Maat_command.cpp
View File

@@ -1403,7 +1403,7 @@ const char* find_Nth_column(const char* line, int Nth, int* column_len)
break;
}
}
if(start==0)
if(start==end)
{
return NULL;
}
@@ -1456,14 +1456,57 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p)
p->table_line=rewrite_line;
return;
}
int get_foreign_keys(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger)
void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n_foreign, const char* dir, void* logger)
{
int ret=0, table_id=0, i=0, j=0;
int foreign_key_size=0;
int rule_with_foreign_key=0;
int ret=0, i=0;
const char* p_foreign=NULL;
struct _Maat_table_info_t* p_table=NULL;
int foreign_key_size=0;
struct stat file_info;
p_rule->n_foreign=n_foreign;
p_rule->f_keys=(struct foreign_key*)calloc(sizeof(struct foreign_key),n_foreign);
for(i=0; i<n_foreign; i++)
{
p_foreign=find_Nth_column(p_rule->table_line, foreign_columns[i], &foreign_key_size);
if(p_foreign==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
, "Get %s,%d foreign keys failed: No %dth column."
, p_rule->table_name, p_rule->rule_id, foreign_columns[i]);
continue;
}
if(0!=strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix)))
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
,"Get %s,%d foreign key failed: Invalid source prefix %s."
, p_rule->table_name, p_rule->rule_id, foreign_columns[i]);
continue;
}
p_rule->f_keys[i].column=foreign_columns[i];
foreign_key_size=foreign_key_size-strlen(foreign_source_prefix);
p_foreign+=strlen(foreign_source_prefix);
if(0!=strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix)))
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
,"%s,%d foreign key prefix %s is discouraged."
,p_rule->table_name, p_rule->rule_id, foreign_columns[i]);
}
p_rule->f_keys[i].key=(char*)calloc(sizeof(char),foreign_key_size+1);
memcpy(p_rule->f_keys[i].key, p_foreign, foreign_key_size);
p_rule->f_keys[i].filename=get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[i].key, dir);
ret=stat(p_rule->f_keys[i].filename, &file_info);
if(ret==0)
{
p_rule->f_keys[i].is_existed=1;
}
}
return;
}
int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger)
{
int ret=0, table_id=0, i=0;
int rule_with_foreign_key=0;
struct _Maat_table_info_t* p_table=NULL;
for(i=0; i<rule_num; i++)
{
if(rule_list[i].table_line==NULL)
@@ -1480,48 +1523,41 @@ int get_foreign_keys(redisContext *ctx, struct serial_rule_t* rule_list, int rul
{
continue;
}
rule_list[i].n_foreign=p_table->n_foreign;
rule_list[i].f_keys=(struct foreign_key*)calloc(sizeof(struct foreign_key), rule_list[i].n_foreign);
for(j=0;j<p_table->n_foreign;j++)
{
p_foreign=find_Nth_column(rule_list[i].table_line, p_table->foreign_columns[j], &foreign_key_size);
if(p_foreign==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
, "Get %s,%d foreign keys failed: No %dth column."
, rule_list[i].table_name, rule_list[i].rule_id, p_table->foreign_columns[j]);
continue;
}
if(0!=strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix)))
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
,"Get %s,%d foreign key failed: Invalid source prefix %s."
, rule_list[i].table_name, rule_list[i].rule_id, p_foreign);
continue;
}
rule_list[i].f_keys[j].column=p_table->foreign_columns[j];
foreign_key_size=foreign_key_size-strlen(foreign_source_prefix);
p_foreign+=strlen(foreign_source_prefix);
if(0!=strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix)))
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
,"%s,%d foreign key prefix %s is discouraged."
, rule_list[i].table_name, rule_list[i].rule_id, p_foreign);
}
rule_list[i].f_keys[j].key=(char*)calloc(sizeof(char),foreign_key_size+1);
memcpy(rule_list[i].f_keys[j].key, p_foreign, foreign_key_size);
rule_list[i].f_keys[j].filename=get_foreign_cont_filename(rule_list[i].table_name, rule_list[i].rule_id, rule_list[i].f_keys[j].key, dir);
ret=stat(rule_list[i].f_keys[j].filename, &file_info);
if(ret==0)
{
rule_list[i].f_keys[j].is_existed=1;
}
}
_get_foregin_keys(rule_list+i, p_table->foreign_columns, p_table->n_foreign, dir, logger);
rule_with_foreign_key++;
}
return rule_with_foreign_key;
}
int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, const char* dir,void *logger)
{
int i=0, j=0, foreign_key_size=0;
int rule_with_foreign_key=0;
const char* p_foreign=NULL;
int n_foreign=0;
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
for(i=0; i<rule_num; i++)
{
j=1;
n_foreign=0;
do{
p_foreign=find_Nth_column(rule_list[i].table_line, j, &foreign_key_size);
if(p_foreign!=NULL&&foreign_key_size>(int)strlen(foreign_source_prefix)&&0==strncmp(p_foreign,foreign_source_prefix, strlen(foreign_source_prefix)))
{
foreign_columns[n_foreign]=j;
n_foreign++;
}
j++;
}while(p_foreign!=NULL&&n_foreign<MAX_FOREIGN_CLMN_NUM);
if(n_foreign>0)
{
_get_foregin_keys(rule_list+i, foreign_columns, n_foreign,dir,logger);
rule_with_foreign_key++;
}
}
return rule_with_foreign_key;
}
struct foreign_conts_track
{
int rule_idx;
@@ -1663,7 +1699,7 @@ void redis_monitor_traverse(long long version,redisContext *c,
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor,"%d of %d rules are empty.",empty_value_num,rule_num);
}
}
ret=get_foreign_keys_define(c, rule_list, rule_num, feather, feather->foreign_cont_dir, logger);
if(ret>0)
{

2
src/entry/Maat_rule.cpp
View File

@@ -32,7 +32,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_2_20180925=1;
int MAAT_FRAME_VERSION_2_2_20180926=1;
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};

5
src/inc_internal/Maat_rule_internal.h
View File

@@ -500,6 +500,11 @@ void maat_stat_output(struct _Maat_feather_t* feather);
redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...);
int get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, long long* new_version, struct serial_rule_t** list,int *update_type, void* logger, int cumulative_off);
int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process);
int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, const char* dir,void *logger);
void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, void *logger);
void rewrite_table_line_with_foreign(struct serial_rule_t*p);
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);
void empty_serial_rules(struct serial_rule_t* rule);
int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger);

20
tools/maat_redis_tool.cpp
View File

@@ -106,6 +106,20 @@ void read_rule_from_redis(redisContext * ctx, long long desire_version, const ch
printf("mkdir %s error\n",output_path);
}
char foreign_files_dir[256];
snprintf(foreign_files_dir, sizeof(foreign_files_dir), "%s/foreign_files/",output_path);
if((access(foreign_files_dir,F_OK)) <0)
{ if((mkdir(foreign_files_dir,S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) < 0)
printf("mkdir %s error\n",foreign_files_dir);
}
ret=get_foreign_keys_by_prefix(ctx, rule_list, rule_num, foreign_files_dir, logger);
if(ret>0)
{
printf("Getting %d foreign content.\n", ret);
get_foreign_conts(ctx, rule_list, rule_num, logger);
}
snprintf(index_path,sizeof(index_path),"%s/full_config_index.%020lld",output_path,version);
index_fp=fopen(index_path,"w");
if(index_fp==NULL)
@@ -116,6 +130,10 @@ void read_rule_from_redis(redisContext * ctx, long long desire_version, const ch
for(i=0;i<rule_num;i++)
{
if(rule_list[i].n_foreign>0)
{
rewrite_table_line_with_foreign(rule_list+i);
}
if(cur_table==NULL||0!=strcmp(cur_table,rule_list[i].table_name))
{
if(table_fp!=NULL)
@@ -223,8 +241,6 @@ int main(int argc, char * argv[])
int redis_port=6379;
int redis_db=0;
strncpy(redis_ip,"127.0.0.1",sizeof(redis_ip));
char table_info[128];
strncpy(table_info,"./table_info.conf",sizeof(table_info));
char dump_dir[128], json_file[128], tmp_iris_path[128];
strncpy(dump_dir,redis_dump_dir,sizeof(dump_dir));
redisContext * ctx=NULL;

12
vendor/CMakeLists.txt vendored
View File

@@ -14,3 +14,15 @@ add_library(gtest STATIC IMPORTED GLOBAL)
set_property(TARGET gtest PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libgtest.a)
set_property(TARGET gtest PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include)
set_property(TARGET gtest PROPERTY INTERFACE_LINK_LIBRARIES pthread)
#### hiredis-vip
#ExternalProject_Add(hiredis-vip PREFIX hiredis-vip
# URL ${CMAKE_CURRENT_SOURCE_DIR}/hiredis-vip-0.3.0.tar.gz
# URL_MD5 e4d4a2928188b11f696d35ca1c52e9a4
# CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
#
#ExternalProject_Get_Property(hiredis-vip INSTALL_DIR)
#file(MAKE_DIRECTORY ${INSTALL_DIR}/include)
#
#add_library(hiredisv STATIC IMPORTED GLOBAL)
#set_property(TARGET gtest PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/libhiredis_vip.a)