diff --git a/bin/conf/sifter/youtube_data.sft b/bin/conf/sifter/youtube_data.sft index ce54327..8610376 100644 --- a/bin/conf/sifter/youtube_data.sft +++ b/bin/conf/sifter/youtube_data.sft @@ -8,5 +8,5 @@ sft_id = 4122 #special character : \r\n = 0x0d0x0a, \x30 = no end , \x0 = null #expect_name = {1}{MEDIA_ID}, {2}{FRAG_UNIT_ID}, {3}{FRAG_UNIT_ABOFFSET}, {4}{FRAG_UNIT_REOFFSET}, {5}{MEDIA_SIZE}, {6}{MEDIA_NAME}, {7}{MEDIA_CONTENT} #extract_location = {61}{C2S_URL}, {0xD1}{S2C_CONTENT-DISPOSITION},{0xE1}{C2S_CONTENT},{0xF1}{S2C_CONTENT} -{1}{MEDIA_ID}={61}{C2S_URL}$End$\x0$\x0$range=$ +{1}{MEDIA_ID}={61}{C2S_URL}$Tag$signature=$\x0$&$ {3}{FRAG_UNIT_ABOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$ diff --git a/src/Makefile b/src/Makefile index a9cac49..ae8d433 100644 --- a/src/Makefile +++ b/src/Makefile @@ -62,6 +62,8 @@ $(TARGET):$(OBJECTS) $(OBJECTSCPP) $(LIB_FILE) #$(CCC) -shared $(CFLAGS) $(OBJECTS) $(LIB) -o $@ $(CCC) $(CFLAGS) $(OBJECTS) $(OBJECTSCPP) $(LIB) $(LDDICTATOR) -o $@ cp $(TARGET) ../bin/ + @echo -e "\033[32mFOR SOQ: make MODE=AV/PIC/VOIP \033[0m" + @echo -e "\033[32mFOR NTC: make PROJECT=K MODE=AV/PIC/VOIP \033[0m" # cp $(TARGET) /home/mesasoft/frag_rssb/ .c.o: diff --git a/src/lib/libsifter.a b/src/lib/libsifter.a index 1595d06..d2ba048 100644 Binary files a/src/lib/libsifter.a and b/src/lib/libsifter.a differ diff --git a/src/support/sifter/bin/libsifter.a b/src/support/sifter/bin/libsifter.a index 1595d06..d2ba048 100644 Binary files a/src/support/sifter/bin/libsifter.a and b/src/support/sifter/bin/libsifter.a differ diff --git a/src/support/sifter/src/inc/Maat_rule.h b/src/support/sifter/src/inc/Maat_rule.h deleted file mode 100644 index 93c96c9..0000000 --- a/src/support/sifter/src/inc/Maat_rule.h +++ /dev/null @@ -1,210 +0,0 @@ - -/* -*****************Maat Network Flow Rule Manage Framework******** -* Maat is the Goddess of truth and justice in ancient Egyptian concept. -* Her feather was the measure that determined whether the souls (considered -* to reside in the heart) of the departed would reach the paradise of afterlife -* successfully. -* Author: zhengchao@iie.ac.cn,MESA -* Version 2015-11-09 digest scan -* NOTE: MUST compile with G++ -* All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018 -********************************************************* -*/ -#ifndef H_MAAT_RULE_H_INCLUDE -#define H_MAAT_RULE_H_INCLUDE -#ifndef __cplusplus -#error("This file should be compiled with C++ compiler") -#endif -#include "stream.h" -enum MAAT_CHARSET -{ - CHARSET_NONE=0, - CHARSET_GBK, - CHARSET_BIG5, - CHARSET_UNICODE, - CHARSET_UTF8, // 4 - CHARSET_BIN //5 -}; -enum MAAT_ACTION -{ - MAAT_ACTION_BLOCK=0, - MAAT_ACTION_MONIT, - MAAT_ACTION_WHITE -}; -enum MAAT_POS_TYPE -{ - MAAT_POSTYPE_EXPR=0, - MAAT_POSTYPE_REGEX -}; -typedef void* scan_status_t; -typedef void* stream_para_t; -typedef void* Maat_feather_t; - - -#define MAX_SERVICE_DEFINE_LEN 128 -struct Maat_rule_t -{ - int config_id; - int service_id; - char do_log; - char do_blacklist; - char action; - char resevered; - int serv_def_len; - char service_defined[MAX_SERVICE_DEFINE_LEN]; -}; -#define MAAT_RULE_UPDATE_TYPE_FULL 1 -#define MAAT_RULE_UPDATE_TYPE_INC 2 -typedef void Maat_start_callback_t(int update_type,void* u_para); -typedef void Maat_update_callback_t(int table_id,const char* table_line,void* u_para); -typedef void Maat_finish_callback_t(void* u_para); - - - - - -//--------------------HITTING DETAIL DESCRIPTION BEGIN - -#define MAAT_MAX_HIT_RULE_NUM 8 -#define MAAT_MAX_EXPR_ITEM_NUM 8 -#define MAAT_MAX_HIT_POS_NUM 2048 -#define MAAT_MAX_REGEX_GROUP_NUM 8 -//NOTE position buffer as hitting_regex_pos and hit_pos,are ONLY valid before next scan or Maat_stream_scan_string_end -struct regex_pos_t -{ - int group_num; - int hitting_regex_len; - const char* hitting_regex_pos; - int grouping_len[MAAT_MAX_REGEX_GROUP_NUM]; - const char* grouping_pos[MAAT_MAX_REGEX_GROUP_NUM]; -}; -struct str_pos_t -{ - int hit_len; - const char* hit_pos; -}; -struct sub_item_pos_t -{ - enum MAAT_POS_TYPE ruletype; - int hit_cnt; - union - { - struct regex_pos_t regex_pos[MAAT_MAX_HIT_POS_NUM]; - struct str_pos_t substr_pos[MAAT_MAX_HIT_POS_NUM]; - }; -}; - -struct Maat_region_pos_t -{ - - int region_id; - int sub_item_num; - struct sub_item_pos_t sub_item_pos[MAAT_MAX_EXPR_ITEM_NUM]; -}; - -struct Maat_hit_detail_t -{ - int config_id;//set <0 if half hit; - int hit_region_cnt; - struct Maat_region_pos_t region_pos[MAAT_MAX_HIT_RULE_NUM]; -}; -//--------------------HITTING DETAIL DESCRIPTION END - -//Abondon interface ,left for compatible. -Maat_feather_t Maat_summon_feather(int max_thread_num, - const char* table_info_path, - const char* ful_cfg_dir, - const char* inc_cfg_dir, - void*logger);//MESA_handle_logger -//Abondon interface ,left for compatible. -Maat_feather_t Maat_summon_feather_json(int max_thread_num, - const char* table_info_path, - const char* json_rule, - void* logger); - -Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger); -int Maat_initiate_feather(Maat_feather_t feather); - -enum MAAT_INIT_OPT -{ - MAAT_OPT_SCANDIR_INTERVAL_MS=1, //VALUE is interger,SIZE=sizeof(int). DEFAULT:1,000 milliseconds. - MAAT_OPT_EFFECT_INVERVAL_MS, //VALUE is interger,SIZE=sizeof(int). DEFAULT:60,000 milliseconds. - MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_INC_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_JSON_FILE_PATH, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_STAT_ON, //VALUE is indifferent,SIZE is indifferent.MAAT_OPT_STAT_FILE_PATH must be set.Default: stat OFF. - MAAT_OPT_PERF_ON, //VALUE is indifferent,SIZE is indifferent.MAAT_OPT_STAT_FILE_PATH must be set.Default: stat OFF. - MAAT_OPT_STAT_FILE_PATH, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_SCAN_DETAIL //VALUE is interger,SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping; - // 2 return hit pos and regex grouping pos;DEFAULT:2 -}; -//return -1 if failed, return 0 on success; -int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); -void Maat_burn_feather(Maat_feather_t feather); - -//return table_id(>=0) if success,otherwise return -1; -int Maat_table_register(Maat_feather_t feather,const char* table_name); -//return 1 if success,otherwise return -1 incase invalid table_id or registed function number exceed 32; -int Maat_table_callback_register(Maat_feather_t feather,short table_id, - Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para - Maat_update_callback_t *update,//table line ,u_para - Maat_finish_callback_t *finish,//u_para - void* u_para); - -enum MAAT_SCAN_OPT -{ - MAAT_SET_SCAN_DISTRICT=1 //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. -}; -//return 0 if success, return -1 when failed; -int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size); - -//Return hit rule number, return -1 when error occurs,return -2 when hit current region -//mid MUST set NULL before fist call -int Maat_scan_intval(Maat_feather_t feather,int table_id - ,unsigned int intval - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num); -int Maat_scan_addr(Maat_feather_t feather,int table_id - ,struct ipaddr* addr - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num); -int Maat_scan_proto_addr(Maat_feather_t feather,int table_id - ,struct ipaddr* addr,unsigned short int proto - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num); -int Maat_full_scan_string(Maat_feather_t feather,int table_id - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int* found_pos,int rule_num - ,scan_status_t* mid,int thread_num); -//hite_detail could be NULL if unconcern -int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num - ,int* detail_ret,scan_status_t* mid,int thread_num); - -stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num); -int Maat_stream_scan_string(stream_para_t* stream_para - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int* found_pos,int rule_num - ,scan_status_t* mid); -//hited_detail could be NULL if unconcern -int Maat_stream_scan_string_detail(stream_para_t* stream_para - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num - ,int* detail_ret,scan_status_t* mid); -void Maat_stream_scan_string_end(stream_para_t* stream_para); - -stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num); -int Maat_stream_scan_digest(stream_para_t* stream_para - ,const char* data,int data_len,unsigned long long offset - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t* mid); -void Maat_stream_scan_digest_end(stream_para_t* stream_para); - - - -void Maat_clean_status(scan_status_t* mid); - -#endif // H_MAAT_RULE_H_INCLUDE - diff --git a/src/support/sifter/src/inc/Maat_rule.h.bak b/src/support/sifter/src/inc/Maat_rule.h.bak deleted file mode 100644 index 9a518ac..0000000 --- a/src/support/sifter/src/inc/Maat_rule.h.bak +++ /dev/null @@ -1,210 +0,0 @@ - -/* -*****************Maat Network Flow Rule Manage Framework******** -* Maat is the Goddess of truth and justice in ancient Egyptian concept. -* Her feather was the measure that determined whether the souls (considered -* to reside in the heart) of the departed would reach the paradise of afterlife -* successfully. -* Author: zhengchao@iie.ac.cn,MESA -* Version 2015-11-09 digest scan -* NOTE: MUST compile with G++ -* All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018 -********************************************************* -*/ -#ifndef H_MAAT_RULE_H_INCLUDE -#define H_MAAT_RULE_H_INCLUDE -#ifndef __cplusplus -#error("This file should be compiled with C++ compiler") -#endif -#include "stream.h" -enum MAAT_CHARSET -{ - CHARSET_NONE=0, - CHARSET_GBK, - CHARSET_BIG5, - CHARSET_UNICODE, - CHARSET_UTF8, // 4 - CHARSET_BIN //5 -}; -enum MAAT_ACTION -{ - MAAT_ACTION_BLOCK=0, - MAAT_ACTION_MONIT, - MAAT_ACTION_WHITE -}; -enum MAAT_POS_TYPE -{ - MAAT_POSTYPE_EXPR=0, - MAAT_POSTYPE_REGEX -}; -typedef void* scan_status_t; -typedef void* stream_para_t; -typedef void* Maat_feather_t; - - -#define MAX_SERVICE_DEFINE_LEN 128 -struct Maat_rule_t -{ - int config_id; - int service_id; - char do_log; - char do_blacklist; - char action; - char resevered; - int serv_def_len; - char service_defined[MAX_SERVICE_DEFINE_LEN]; -}; -#define MAAT_RULE_UPDATE_TYPE_FULL 1 -#define MAAT_RULE_UPDATE_TYPE_INC 2 -typedef void Maat_start_callback_t(int update_type,void* u_para); -typedef void Maat_update_callback_t(int table_id,const char* table_line,void* u_para); -typedef void Maat_finish_callback_t(void* u_para); - - - - - -//--------------------HITTING DETAIL DESCRIPTION BEGIN - -#define MAAT_MAX_HIT_RULE_NUM 8 -#define MAAT_MAX_EXPR_ITEM_NUM 8 -#define MAAT_MAX_HIT_POS_NUM 512 -#define MAAT_MAX_REGEX_GROUP_NUM 8 -//NOTE position buffer as hitting_regex_pos and hit_pos,are ONLY valid before next scan or Maat_stream_scan_string_end -struct regex_pos_t -{ - int group_num; - int hitting_regex_len; - const char* hitting_regex_pos; - int grouping_len[MAAT_MAX_REGEX_GROUP_NUM]; - const char* grouping_pos[MAAT_MAX_REGEX_GROUP_NUM]; -}; -struct str_pos_t -{ - int hit_len; - const char* hit_pos; -}; -struct sub_item_pos_t -{ - enum MAAT_POS_TYPE ruletype; - int hit_cnt; - union - { - struct regex_pos_t regex_pos[MAAT_MAX_HIT_POS_NUM]; - struct str_pos_t substr_pos[MAAT_MAX_HIT_POS_NUM]; - }; -}; - -struct Maat_region_pos_t -{ - - int region_id; - int sub_item_num; - struct sub_item_pos_t sub_item_pos[MAAT_MAX_EXPR_ITEM_NUM]; -}; - -struct Maat_hit_detail_t -{ - int config_id;//set <0 if half hit; - int hit_region_cnt; - struct Maat_region_pos_t region_pos[MAAT_MAX_HIT_RULE_NUM]; -}; -//--------------------HITTING DETAIL DESCRIPTION END - -//Abondon interface ,left for compatible. -Maat_feather_t Maat_summon_feather(int max_thread_num, - const char* table_info_path, - const char* ful_cfg_dir, - const char* inc_cfg_dir, - void*logger);//MESA_handle_logger -//Abondon interface ,left for compatible. -Maat_feather_t Maat_summon_feather_json(int max_thread_num, - const char* table_info_path, - const char* json_rule, - void* logger); - -Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger); -int Maat_initiate_feather(Maat_feather_t feather); - -enum MAAT_INIT_OPT -{ - MAAT_OPT_SCANDIR_INTERVAL_MS=1, //VALUE is interger,SIZE=sizeof(int). DEFAULT:1,000 milliseconds. - MAAT_OPT_EFFECT_INVERVAL_MS, //VALUE is interger,SIZE=sizeof(int). DEFAULT:60,000 milliseconds. - MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_INC_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_JSON_FILE_PATH, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_STAT_ON, //VALUE is indifferent,SIZE is indifferent.MAAT_OPT_STAT_FILE_PATH must be set.Default: stat OFF. - MAAT_OPT_PERF_ON, //VALUE is indifferent,SIZE is indifferent.MAAT_OPT_STAT_FILE_PATH must be set.Default: stat OFF. - MAAT_OPT_STAT_FILE_PATH, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_SCAN_DETAIL //VALUE is interger,SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping; - // 2 return hit pos and regex grouping pos;DEFAULT:2 -}; -//return -1 if failed, return 0 on success; -int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); -void Maat_burn_feather(Maat_feather_t feather); - -//return table_id(>=0) if success,otherwise return -1; -int Maat_table_register(Maat_feather_t feather,const char* table_name); -//return 1 if success,otherwise return -1 incase invalid table_id or registed function number exceed 32; -int Maat_table_callback_register(Maat_feather_t feather,short table_id, - Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para - Maat_update_callback_t *update,//table line ,u_para - Maat_finish_callback_t *finish,//u_para - void* u_para); - -enum MAAT_SCAN_OPT -{ - MAAT_SET_SCAN_DISTRICT=1 //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default. -}; -//return 0 if success, return -1 when failed; -int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size); - -//Return hit rule number, return -1 when error occurs,return -2 when hit current region -//mid MUST set NULL before fist call -int Maat_scan_intval(Maat_feather_t feather,int table_id - ,unsigned int intval - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num); -int Maat_scan_addr(Maat_feather_t feather,int table_id - ,struct ipaddr* addr - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num); -int Maat_scan_proto_addr(Maat_feather_t feather,int table_id - ,struct ipaddr* addr,unsigned short int proto - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num); -int Maat_full_scan_string(Maat_feather_t feather,int table_id - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int* found_pos,int rule_num - ,scan_status_t* mid,int thread_num); -//hite_detail could be NULL if unconcern -int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num - ,int* detail_ret,scan_status_t* mid,int thread_num); - -stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num); -int Maat_stream_scan_string(stream_para_t* stream_para - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int* found_pos,int rule_num - ,scan_status_t* mid); -//hited_detail could be NULL if unconcern -int Maat_stream_scan_string_detail(stream_para_t* stream_para - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num - ,int* detail_ret,scan_status_t* mid); -void Maat_stream_scan_string_end(stream_para_t* stream_para); - -stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num); -int Maat_stream_scan_digest(stream_para_t* stream_para - ,const char* data,int data_len,unsigned long long offset - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t* mid); -void Maat_stream_scan_digest_end(stream_para_t* stream_para); - - - -void Maat_clean_status(scan_status_t* mid); - -#endif // H_MAAT_RULE_H_INCLUDE - diff --git a/src/support/sifter/src/libsifter.a b/src/support/sifter/src/libsifter.a index 1595d06..d2ba048 100644 Binary files a/src/support/sifter/src/libsifter.a and b/src/support/sifter/src/libsifter.a differ