diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a5e6477..dd69c40 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,9 +1,9 @@ -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 3.5) project(maatframe) set(MAAT_FRAME_MAJOR_VERSION 2) set(MAAT_FRAME_MINOR_VERSION 2) -set(MAAT_FRAME_PATCH_VERSION 20180923) +set(MAAT_FRAME_PATCH_VERSION 20180926) set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION}) message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index a7d581c..9e139a9 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1403,7 +1403,7 @@ const char* find_Nth_column(const char* line, int Nth, int* column_len) break; } } - if(start==0) + if(start==end) { return NULL; } @@ -1456,14 +1456,57 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p) p->table_line=rewrite_line; return; } -int get_foreign_keys(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger) +void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n_foreign, const char* dir, void* logger) { - int ret=0, table_id=0, i=0, j=0; - int foreign_key_size=0; - int rule_with_foreign_key=0; + int ret=0, i=0; const char* p_foreign=NULL; - struct _Maat_table_info_t* p_table=NULL; + int foreign_key_size=0; struct stat file_info; + p_rule->n_foreign=n_foreign; + p_rule->f_keys=(struct foreign_key*)calloc(sizeof(struct foreign_key),n_foreign); + for(i=0; itable_line, foreign_columns[i], &foreign_key_size); + if(p_foreign==NULL) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor + , "Get %s,%d foreign keys failed: No %dth column." + , p_rule->table_name, p_rule->rule_id, foreign_columns[i]); + continue; + } + if(0!=strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor + ,"Get %s,%d foreign key failed: Invalid source prefix %s." + , p_rule->table_name, p_rule->rule_id, foreign_columns[i]); + continue; + } + p_rule->f_keys[i].column=foreign_columns[i]; + foreign_key_size=foreign_key_size-strlen(foreign_source_prefix); + p_foreign+=strlen(foreign_source_prefix); + if(0!=strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) + { + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor + ,"%s,%d foreign key prefix %s is discouraged." + ,p_rule->table_name, p_rule->rule_id, foreign_columns[i]); + } + p_rule->f_keys[i].key=(char*)calloc(sizeof(char),foreign_key_size+1); + memcpy(p_rule->f_keys[i].key, p_foreign, foreign_key_size); + + p_rule->f_keys[i].filename=get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[i].key, dir); + ret=stat(p_rule->f_keys[i].filename, &file_info); + if(ret==0) + { + p_rule->f_keys[i].is_existed=1; + } + } + return; +} +int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger) +{ + int ret=0, table_id=0, i=0; + int rule_with_foreign_key=0; + struct _Maat_table_info_t* p_table=NULL; for(i=0; in_foreign; - rule_list[i].f_keys=(struct foreign_key*)calloc(sizeof(struct foreign_key), rule_list[i].n_foreign); - for(j=0;jn_foreign;j++) - { - p_foreign=find_Nth_column(rule_list[i].table_line, p_table->foreign_columns[j], &foreign_key_size); - if(p_foreign==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor - , "Get %s,%d foreign keys failed: No %dth column." - , rule_list[i].table_name, rule_list[i].rule_id, p_table->foreign_columns[j]); - continue; - } - if(0!=strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor - ,"Get %s,%d foreign key failed: Invalid source prefix %s." - , rule_list[i].table_name, rule_list[i].rule_id, p_foreign); - continue; - } - rule_list[i].f_keys[j].column=p_table->foreign_columns[j]; - foreign_key_size=foreign_key_size-strlen(foreign_source_prefix); - p_foreign+=strlen(foreign_source_prefix); - if(0!=strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor - ,"%s,%d foreign key prefix %s is discouraged." - , rule_list[i].table_name, rule_list[i].rule_id, p_foreign); - } - rule_list[i].f_keys[j].key=(char*)calloc(sizeof(char),foreign_key_size+1); - memcpy(rule_list[i].f_keys[j].key, p_foreign, foreign_key_size); - - rule_list[i].f_keys[j].filename=get_foreign_cont_filename(rule_list[i].table_name, rule_list[i].rule_id, rule_list[i].f_keys[j].key, dir); - ret=stat(rule_list[i].f_keys[j].filename, &file_info); - if(ret==0) - { - rule_list[i].f_keys[j].is_existed=1; - } - } + _get_foregin_keys(rule_list+i, p_table->foreign_columns, p_table->n_foreign, dir, logger); rule_with_foreign_key++; } return rule_with_foreign_key; } +int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, const char* dir,void *logger) +{ + int i=0, j=0, foreign_key_size=0; + int rule_with_foreign_key=0; + const char* p_foreign=NULL; + + int n_foreign=0; + int foreign_columns[MAX_FOREIGN_CLMN_NUM]; + for(i=0; i(int)strlen(foreign_source_prefix)&&0==strncmp(p_foreign,foreign_source_prefix, strlen(foreign_source_prefix))) + { + foreign_columns[n_foreign]=j; + n_foreign++; + } + j++; + }while(p_foreign!=NULL&&n_foreign0) + { + _get_foregin_keys(rule_list+i, foreign_columns, n_foreign,dir,logger); + rule_with_foreign_key++; + } + } + return rule_with_foreign_key; +} + struct foreign_conts_track { int rule_idx; @@ -1663,7 +1699,7 @@ void redis_monitor_traverse(long long version,redisContext *c, { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor,"%d of %d rules are empty.",empty_value_num,rule_num); } - ret=get_foreign_keys(c, rule_list, rule_num, feather, feather->foreign_cont_dir, logger); + ret=get_foreign_keys_define(c, rule_list, rule_num, feather, feather->foreign_cont_dir, logger); if(ret>0) { get_foreign_conts(c, rule_list, rule_num, logger); diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 65180f5..5f25c35 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -32,7 +32,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_2_20180925=1; +int MAAT_FRAME_VERSION_2_2_20180926=1; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 8f9c0cf..e2a1f53 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -500,6 +500,11 @@ void maat_stat_output(struct _Maat_feather_t* feather); redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); int get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, long long* new_version, struct serial_rule_t** list,int *update_type, void* logger, int cumulative_off); int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process); +int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, const char* dir,void *logger); +void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, void *logger); +void rewrite_table_line_with_foreign(struct serial_rule_t*p); + + void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout); void empty_serial_rules(struct serial_rule_t* rule); int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger); diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 8b0a53c..4e731cf 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -106,6 +106,20 @@ void read_rule_from_redis(redisContext * ctx, long long desire_version, const ch printf("mkdir %s error\n",output_path); } + char foreign_files_dir[256]; + snprintf(foreign_files_dir, sizeof(foreign_files_dir), "%s/foreign_files/",output_path); + if((access(foreign_files_dir,F_OK)) <0) + + { if((mkdir(foreign_files_dir,S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) < 0) + printf("mkdir %s error\n",foreign_files_dir); + + } + ret=get_foreign_keys_by_prefix(ctx, rule_list, rule_num, foreign_files_dir, logger); + if(ret>0) + { + printf("Getting %d foreign content.\n", ret); + get_foreign_conts(ctx, rule_list, rule_num, logger); + } snprintf(index_path,sizeof(index_path),"%s/full_config_index.%020lld",output_path,version); index_fp=fopen(index_path,"w"); if(index_fp==NULL) @@ -116,6 +130,10 @@ void read_rule_from_redis(redisContext * ctx, long long desire_version, const ch for(i=0;i0) + { + rewrite_table_line_with_foreign(rule_list+i); + } if(cur_table==NULL||0!=strcmp(cur_table,rule_list[i].table_name)) { if(table_fp!=NULL) @@ -223,8 +241,6 @@ int main(int argc, char * argv[]) int redis_port=6379; int redis_db=0; strncpy(redis_ip,"127.0.0.1",sizeof(redis_ip)); - char table_info[128]; - strncpy(table_info,"./table_info.conf",sizeof(table_info)); char dump_dir[128], json_file[128], tmp_iris_path[128]; strncpy(dump_dir,redis_dump_dir,sizeof(dump_dir)); redisContext * ctx=NULL; diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index c1b17d5..2d8dc72 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -13,4 +13,16 @@ file(MAKE_DIRECTORY ${INSTALL_DIR}/include) add_library(gtest STATIC IMPORTED GLOBAL) set_property(TARGET gtest PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libgtest.a) set_property(TARGET gtest PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) -set_property(TARGET gtest PROPERTY INTERFACE_LINK_LIBRARIES pthread) \ No newline at end of file +set_property(TARGET gtest PROPERTY INTERFACE_LINK_LIBRARIES pthread) + +#### hiredis-vip +#ExternalProject_Add(hiredis-vip PREFIX hiredis-vip +# URL ${CMAKE_CURRENT_SOURCE_DIR}/hiredis-vip-0.3.0.tar.gz +# URL_MD5 e4d4a2928188b11f696d35ca1c52e9a4 +# CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}) +# +#ExternalProject_Get_Property(hiredis-vip INSTALL_DIR) +#file(MAKE_DIRECTORY ${INSTALL_DIR}/include) +# +#add_library(hiredisv STATIC IMPORTED GLOBAL) +#set_property(TARGET gtest PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/libhiredis_vip.a)