diff --git a/bin/conf/sifter/iqiyi_data.sft b/bin/conf/sifter/iqiyi_data.sft deleted file mode 100644 index fff3ec4..0000000 --- a/bin/conf/sifter/iqiyi_data.sft +++ /dev/null @@ -1,12 +0,0 @@ -[INFOR] -sft_id = 3122 - -[EXTRACT] -#[expect_type][expect_name] = extract_location$extract_method$start$mid$end$ -#expect_type and expect_name must be one -#sifter_method = Tag | RawText | Boundary | Reposition -#special character : \r\n = 0x0d0x0a, \x30 = no end , \x0 = null -#expect_name = {1}{MEDIA_ID}, {2}{FRAG_UNIT_ID}, {3}{FRAG_UNIT_ABOFFSET}, {4}{FRAG_UNIT_REOFFSET}, {5}{MEDIA_SIZE}, {6}{MEDIA_NAME}, {7}{MEDIA_CONTENT} -#extract_location = {61}{C2S_URL}, {0xD1}{S2C_CONTENT-DISPOSITION},{0xE1}{C2S_CONTENT},{0xF1}{S2C_CONTENT} -{1}{MEDIA_ID}={61}{C2S_URL}$End$\x0$\x0$f4v$ -{4}{FRAG_UNIT_ABOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$ diff --git a/bin/conf/sifter/sifter.json b/bin/conf/sifter/sifter.json index c78b10b..07ef58f 100644 --- a/bin/conf/sifter/sifter.json +++ b/bin/conf/sifter/sifter.json @@ -5,12 +5,12 @@ [ { "compile_id": 1, - "service": 2221, - "action": 2, + "service": 5222, + "action": 1, "do_blacklist": 1, - "do_log": 1, + "do_log": 0, "effective_rage": 0, - "user_region": "FRAG_UNIT_ID", + "user_region": "MEDIA_ID", "is_valid": "yes", "groups": [ { @@ -20,7 +20,7 @@ "table_name": "SIFTER_START", "table_type": "string", "table_content": { - "keywords": "http://", + "keywords": "/vmind", "expr_type": "none", "match_method": "sub", "format": "uncase plain" @@ -32,12 +32,12 @@ }, { "compile_id": 2, - "service": 2221, - "action": 4, + "service": 5222, + "action": 2, "do_blacklist": 1, - "do_log": 2, + "do_log": 1, "effective_rage": 0, - "user_region": "FRAG_UNIT_REOFFSET", + "user_region": "FRAG_UNIT_ID", "is_valid": "yes", "groups": [ { @@ -47,7 +47,7 @@ "table_name": "SIFTER_START", "table_type": "string", "table_content": { - "keywords": "&ts_seg_no=", + "keywords": "/vmind", "expr_type": "none", "match_method": "sub", "format": "uncase plain" @@ -113,70 +113,16 @@ }, { "compile_id": 5, - "service": 3122, - "action": 4, - "do_blacklist": 4, - "do_log": 1, - "effective_rage": 0, - "user_region": "FRAG_UNIT_ABOFFSET", - "is_valid": "yes", - "groups": [ - { - "group_name": "4_start", - "regions": [ - { - "table_name": "SIFTER_START", - "table_type": "string", - "table_content": { - "keywords": "range=", - "expr_type": "none", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - } - ] - }, -{ - "compile_id": 6, - "service": 1122, - "action": 4, - "do_blacklist": 5, - "do_log": 1, - "effective_rage": 0, - "user_region": "FRAG_UNIT_REOFFSET", - "is_valid": "yes", - "groups": [ - { - "group_name": "5_start", - "regions": [ - { - "table_name": "SIFTER_START", - "table_type": "string", - "table_content": { - "keywords": "Frag", - "expr_type": "none", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - } - ] - }, -{ - "compile_id": 7, "service": 2121, "action": 2, - "do_blacklist": 7, + "do_blacklist": 4, "do_log": 1, "effective_rage": 0, "user_region": "FRAG_UNIT_ID", "is_valid": "yes", "groups": [ { - "group_name": "6_start", + "group_name": "4_start", "regions": [ { "table_name": "SIFTER_START", @@ -193,17 +139,17 @@ ] }, { - "compile_id": 8, + "compile_id": 6, "service": 2121, "action": 4, - "do_blacklist": 7, + "do_blacklist": 4, "do_log": 2, "effective_rage": 0, "user_region": "FRAG_UNIT_REOFFSET", "is_valid": "yes", "groups": [ { - "group_name": "7_start", + "group_name": "5_start", "regions": [ { "table_name": "SIFTER_START", @@ -220,13 +166,67 @@ ] }, { - "compile_id": 9, - "service": 5222, - "action": 1, - "do_blacklist": 8, - "do_log": 0, + "compile_id": 7, + "service": 1122, + "action": 4, + "do_blacklist": 5, + "do_log": 1, "effective_rage": 0, - "user_region": "MEDIA_ID", + "user_region": "FRAG_UNIT_REOFFSET", + "is_valid": "yes", + "groups": [ + { + "group_name": "6_start", + "regions": [ + { + "table_name": "SIFTER_START", + "table_type": "string", + "table_content": { + "keywords": "Frag", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, +{ + "compile_id": 8, + "service": 2221, + "action": 2, + "do_blacklist": 7, + "do_log": 1, + "effective_rage": 0, + "user_region": "FRAG_UNIT_ID", + "is_valid": "yes", + "groups": [ + { + "group_name": "7_start", + "regions": [ + { + "table_name": "SIFTER_START", + "table_type": "string", + "table_content": { + "keywords": "http://", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, +{ + "compile_id": 9, + "service": 2221, + "action": 4, + "do_blacklist": 7, + "do_log": 2, + "effective_rage": 0, + "user_region": "FRAG_UNIT_REOFFSET", "is_valid": "yes", "groups": [ { @@ -236,7 +236,7 @@ "table_name": "SIFTER_START", "table_type": "string", "table_content": { - "keywords": "/vmind", + "keywords": "&ts_seg_no=", "expr_type": "none", "match_method": "sub", "format": "uncase plain" @@ -248,12 +248,12 @@ }, { "compile_id": 10, - "service": 5222, - "action": 2, + "service": 4122, + "action": 3, "do_blacklist": 8, "do_log": 1, "effective_rage": 0, - "user_region": "FRAG_UNIT_ID", + "user_region": "FRAG_UNIT_ABOFFSET", "is_valid": "yes", "groups": [ { @@ -263,7 +263,7 @@ "table_name": "SIFTER_START", "table_type": "string", "table_content": { - "keywords": "/vmind", + "keywords": "range=", "expr_type": "none", "match_method": "sub", "format": "uncase plain" diff --git a/bin/conf/sifter/urloffset_data.sft b/bin/conf/sifter/urloffset_data.sft deleted file mode 100644 index f18b19b..0000000 --- a/bin/conf/sifter/urloffset_data.sft +++ /dev/null @@ -1,12 +0,0 @@ -[INFOR] -sft_id = 3122 - -[EXTRACT] -#[expect_type][expect_name] = extract_location$extract_method$start$mid$end$ -#expect_type and expect_name must be one -#sifter_method = Tag | RawText | Boundary | Reposition -#special character : \r\n = 0x0d0x0a, \x30 = no end , \x0 = null -#expect_name = {1}{MEDIA_ID}, {2}{FRAG_UNIT_ID}, {3}{FRAG_UNIT_ABOFFSET}, {4}{FRAG_UNIT_REOFFSET}, {5}{MEDIA_SIZE}, {6}{MEDIA_NAME}, {7}{MEDIA_CONTENT} -#extract_location = {61}{C2S_URL}, {0xD1}{S2C_CONTENT-DISPOSITION},{0xE1}{C2S_CONTENT},{0xF1}{S2C_CONTENT} -{1}{MEDIA_ID}={61}{C2S_URL}$End$\x0$\x0$f4v$ -{4}{FRAG_UNIT_REOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$ diff --git a/bin/conf/sifter/youtube_data.sft b/bin/conf/sifter/youtube_data.sft index 518173b..ce54327 100644 --- a/bin/conf/sifter/youtube_data.sft +++ b/bin/conf/sifter/youtube_data.sft @@ -9,4 +9,4 @@ sft_id = 4122 #expect_name = {1}{MEDIA_ID}, {2}{FRAG_UNIT_ID}, {3}{FRAG_UNIT_ABOFFSET}, {4}{FRAG_UNIT_REOFFSET}, {5}{MEDIA_SIZE}, {6}{MEDIA_NAME}, {7}{MEDIA_CONTENT} #extract_location = {61}{C2S_URL}, {0xD1}{S2C_CONTENT-DISPOSITION},{0xE1}{C2S_CONTENT},{0xF1}{S2C_CONTENT} {1}{MEDIA_ID}={61}{C2S_URL}$End$\x0$\x0$range=$ -{4}{FRAG_UNIT_REOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$ +{3}{FRAG_UNIT_ABOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$ diff --git a/frag_rssb安装使用手册.docx b/frag_rssb安装使用手册.docx index 729095f..4e34160 100644 Binary files a/frag_rssb安装使用手册.docx and b/frag_rssb安装使用手册.docx differ diff --git a/src/AV_interface.h b/src/AV_interface.h index 13996a7..74bd8f6 100644 --- a/src/AV_interface.h +++ b/src/AV_interface.h @@ -36,7 +36,7 @@ #define FILE_IOS 0xA3 #define FILE_ANDRIOD 0xA4 #define FILE_APP 0xA5 -#define FILE_FRAG 0xA6 /*ËéƬԤ¸æʶ±ðµÄ,ÀýÈçurlÀïÃæ´øÓÐÆ«ÒÆÁ¿µÄ£¬ÀýÈçiqiyi*/ +#define FILE_REQ_FRAG 0xA6 /*ËéƬԤ¸æʶ±ðµÄ,ÀýÈçurlÀïÃæ´øÓÐÆ«ÒÆÁ¿µÄ£¬ÀýÈçiqiyi*/ #define FILE_MAYBE_FRAG 0xA7 /*Ç°¶Ë»Ø´«µÄ¿ÉÄÜÊÇËéƬµÄÒôÊÓƵ*/ #define FILE_AV 0x00 /*ËéƬԤ¸æʶ±ðµÄýÌåÀàÐÍת»»Îª0x00*/ diff --git a/src/frag_av.c b/src/frag_av.c index 01b68ec..80a0669 100644 --- a/src/frag_av.c +++ b/src/frag_av.c @@ -751,7 +751,7 @@ void media_stat(media_t* mdi, frag_unit_t* frg_unit) atomic_inc(&frag_rssb.data_info[RSSB_OSMF_TO_OTHER][TOTAL_PKTS]); } break; - case FILE_FRAG: + case FILE_REQ_FRAG: if(frg_unit->service_id!=0) { atomic_inc(&frag_rssb.data_info[RSSB_RECV_FRAG_MEDIA][TOTAL_PKTS]); @@ -794,7 +794,7 @@ void media_byte_stat(media_t* mdi, frag_unit_t* frg_unit, frag_in_t* frg) } break; - case FILE_FRAG: + case FILE_REQ_FRAG: if(frg_unit!=NULL && frg_unit->service_id!=0) { atomic_add(&frag_rssb.data_info[RSSB_RECV_FRAG_MEDIA][TOTAL_BYTES],frg->datalen); @@ -900,10 +900,10 @@ int media_removal(media_t* mdi, frag_unit_t* frg_unit, frag_in_t* frg, frag_ivi_ frg->seq = mdi->re_offset; frg->offset_in = frg->offset; frag_stat = 1; - } + } else { - if(FILE_FRAG==mdi->media_type && NULL!=frg_unit) + if(FILE_REQ_FRAG==mdi->media_type && NULL!=frg_unit) { frg->seq = frg_unit->re_offset; frg->offset += frg_unit->ab_offset; @@ -1298,7 +1298,7 @@ long converge_mediainfo_search_cb(void *data, const uint8_t *key, uint size, voi int is_frag(uint8_t media_type) { - if(media_type==FILE_OSMF || media_type==FILE_HLS || media_type==FILE_FRAG || media_type==FILE_MAYBE_FRAG ||media_type==FILE_IOS|| media_type==FILE_ANDRIOD|| media_type==FILE_APP) + if(media_type==FILE_OSMF || media_type==FILE_HLS || media_type==FILE_REQ_FRAG || media_type==FILE_MAYBE_FRAG ||media_type==FILE_IOS|| media_type==FILE_ANDRIOD|| media_type==FILE_APP) { return 1; } diff --git a/src/frag_reassembly.c b/src/frag_reassembly.c index ed0fde7..4c87e60 100644 --- a/src/frag_reassembly.c +++ b/src/frag_reassembly.c @@ -481,7 +481,7 @@ void set_media_service_type(media_t* mdi) mdi->media_service_type = MEDIA_SERVICE_TYPE_FRAG; mdi->media_len = 0; } - else if(FILE_FRAG==mdi->media_type) + else if(FILE_REQ_FRAG==mdi->media_type) { mdi->media_service_type = MEDIA_SERVICE_TYPE_FRAG; } @@ -709,7 +709,7 @@ void free_frag_unit(void* data) { case FILE_HLS: case FILE_OSMF: - case FILE_FRAG: + case FILE_REQ_FRAG: case FILE_MAYBE_FRAG: frag_redis_index_twice(frg_unit); break; @@ -1045,15 +1045,15 @@ long media_preproc_cb(void *data, const uint8_t *key, uint size, void *user_arg) } /*offset==0, log*/ - if(0==frg->offset) - { - if(!FLAG_TEST(mdi->flag, PROG_OFFSET_ZERO)) - { - atomic_inc(&g_frag_stat.media_stat[LOG_MEDIA_OFFSET_ZERO]); - FLAG_SET(mdi->flag, PROG_OFFSET_ZERO); - } - create_media_write_to_log(mdi, MEDIA_OFFSET_ZERO, NULL); - } + if(0==frg->offset) + { + if(!FLAG_TEST(mdi->flag, PROG_OFFSET_ZERO)) + { + atomic_inc(&g_frag_stat.media_stat[LOG_MEDIA_OFFSET_ZERO]); + FLAG_SET(mdi->flag, PROG_OFFSET_ZERO); + } + create_media_write_to_log(mdi, MEDIA_OFFSET_ZERO, NULL); + } /*record maxoffset*/ mdi->maxoffset = MAX(frg->offset, mdi->maxoffset); @@ -1335,7 +1335,7 @@ long get_media(void *data, const uint8_t *key, uint size, void *user_arg) } /*±ãÓÚͳ¼Æfor data*/ /*³ýÁËHLS OSMF£¬ÆäËûËéƬЭÒémedia_typeÉèÖÃΪ´«Í³½ÚÄ¿µÄmediatype*/ - if(mdi->media_type==FILE_FRAG) + if(mdi->media_type==FILE_REQ_FRAG) { media_info->media_type = FILE_AV; media_info->prog_len = 0; diff --git a/src/main.c b/src/main.c index 43052af..b1f77c3 100644 --- a/src/main.c +++ b/src/main.c @@ -42,9 +42,9 @@ const char* frag_rssb_version = "2018-08-13T09:00:00"; const char* frag_rssb_version_time = "2018-08-13T09:00:00"; const char* frag_rssb_version_des = "MESA@iie rssb_maskey"; -int FRAG_RSSB_VERSION_1_0_20181211 = 0; -const char* frag_rssb_version_time_in = "2018-12-11"; -const char* frag_rssb_version_des_in = "resp_checkresult_search_media_cb add K_PROJECT"; +int FRAG_RSSB_VERSION_1_0_20190121 = 0; +const char* frag_rssb_version_time_in = "2019-01-21"; +const char* frag_rssb_version_des_in = "support youtube"; void frag_rssb_history() { //2015.11.15 v1.0 create the project @@ -218,7 +218,7 @@ void frag_rssb_history() //2018.09.13 v4.0 //1.frag removal //2018.09.20 v4.0//1 voip_fulllog add voice_dir opt //2018.09.27 v4.0 //1. hard balance - //2018.10.08 v4.0//1.add send_voip_full_json_log for K_PROJECT;2.voip_fulllog and voip_surveylog add voice_dir and cap_ip opt + //2018.10.08 v4.0//1.add send_voip_full_json_log for K_PROJECT;2.voip_fulllog and voip_surveylog add voice_dir and cap_ip opt //2018.10.22 v4.0//alter g_sip_opt_type and add enum sip_opt_index //2018.11.02 v4.0//add timer_on //2018.11.05 v4.0//alter voip save_media frg_len>headerlen @@ -226,7 +226,8 @@ void frag_rssb_history() //2018.12.07 v4.0//unixsocket set bufsize timeout in configure //2018.12.10 v4.0//1.recv_msg_fd not unix 2.MsgPort //2018.12.11 v4.0//1.file mediatype - //2018.12.11 v4.0//resp_checkresult_search_media_cb add K_PROJECT,and delete send_json_log + //2018.12.11 v4.0//resp_checkresult_search_media_cb add K_PROJECT,and delete send_json_log + //2019.01.21 v4.0//support youtube; update req_frag and template } frag_rssb_parameter_t g_frag_run;