gfwleak/av-frag-rssb
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持youtube

Browse Source
This commit is contained in:
lishu
2019-01-28 14:39:51 +08:00
parent b78826a8d6
commit 48379bddd3
9 changed files with 105 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
bin/conf/sifter/iqiyi_data.sft
View File

@@ -1,12 +0,0 @@
[INFOR]
sft_id = 3122
[EXTRACT]
#[expect_type][expect_name] = extract_location$extract_method$start$mid$end$
#expect_type and expect_name must be one
#sifter_method = Tag | RawText | Boundary | Reposition
#special character : \r\n = 0x0d0x0a, \x30 = no end , \x0 = null
#expect_name = {1}{MEDIA_ID}, {2}{FRAG_UNIT_ID}, {3}{FRAG_UNIT_ABOFFSET}, {4}{FRAG_UNIT_REOFFSET}, {5}{MEDIA_SIZE}, {6}{MEDIA_NAME}, {7}{MEDIA_CONTENT}
#extract_location = {61}{C2S_URL}, {0xD1}{S2C_CONTENT-DISPOSITION},{0xE1}{C2S_CONTENT},{0xF1}{S2C_CONTENT}
{1}{MEDIA_ID}={61}{C2S_URL}$End$\x0$\x0$f4v$
{4}{FRAG_UNIT_ABOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$

160
bin/conf/sifter/sifter.json
View File

@@ -5,12 +5,12 @@
[
{
"compile_id": 1,
"service": 2221,
"action": 2,
"service": 5222,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"do_log": 0,
"effective_rage": 0,
"user_region": "FRAG_UNIT_ID",
"user_region": "MEDIA_ID",
"is_valid": "yes",
"groups": [
{
@@ -20,7 +20,7 @@
"table_name": "SIFTER_START",
"table_type": "string",
"table_content": {
"keywords": "http://",
"keywords": "/vmind",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
@@ -32,12 +32,12 @@
},
{
"compile_id": 2,
"service": 2221,
"action": 4,
"service": 5222,
"action": 2,
"do_blacklist": 1,
"do_log": 2,
"do_log": 1,
"effective_rage": 0,
"user_region": "FRAG_UNIT_REOFFSET",
"user_region": "FRAG_UNIT_ID",
"is_valid": "yes",
"groups": [
{
@@ -47,7 +47,7 @@
"table_name": "SIFTER_START",
"table_type": "string",
"table_content": {
"keywords": "&ts_seg_no=",
"keywords": "/vmind",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
@@ -113,70 +113,16 @@
},
{
"compile_id": 5,
"service": 3122,
"action": 4,
"do_blacklist": 4,
"do_log": 1,
"effective_rage": 0,
"user_region": "FRAG_UNIT_ABOFFSET",
"is_valid": "yes",
"groups": [
{
"group_name": "4_start",
"regions": [
{
"table_name": "SIFTER_START",
"table_type": "string",
"table_content": {
"keywords": "range=",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 6,
"service": 1122,
"action": 4,
"do_blacklist": 5,
"do_log": 1,
"effective_rage": 0,
"user_region": "FRAG_UNIT_REOFFSET",
"is_valid": "yes",
"groups": [
{
"group_name": "5_start",
"regions": [
{
"table_name": "SIFTER_START",
"table_type": "string",
"table_content": {
"keywords": "Frag",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 7,
"service": 2121,
"action": 2,
"do_blacklist": 7,
"do_blacklist": 4,
"do_log": 1,
"effective_rage": 0,
"user_region": "FRAG_UNIT_ID",
"is_valid": "yes",
"groups": [
{
"group_name": "6_start",
"group_name": "4_start",
"regions": [
{
"table_name": "SIFTER_START",
@@ -193,17 +139,17 @@
]
},
{
"compile_id": 8,
"compile_id": 6,
"service": 2121,
"action": 4,
"do_blacklist": 7,
"do_blacklist": 4,
"do_log": 2,
"effective_rage": 0,
"user_region": "FRAG_UNIT_REOFFSET",
"is_valid": "yes",
"groups": [
{
"group_name": "7_start",
"group_name": "5_start",
"regions": [
{
"table_name": "SIFTER_START",
@@ -220,13 +166,67 @@
]
},
{
"compile_id": 9,
"service": 5222,
"action": 1,
"do_blacklist": 8,
"do_log": 0,
"compile_id": 7,
"service": 1122,
"action": 4,
"do_blacklist": 5,
"do_log": 1,
"effective_rage": 0,
"user_region": "MEDIA_ID",
"user_region": "FRAG_UNIT_REOFFSET",
"is_valid": "yes",
"groups": [
{
"group_name": "6_start",
"regions": [
{
"table_name": "SIFTER_START",
"table_type": "string",
"table_content": {
"keywords": "Frag",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 8,
"service": 2221,
"action": 2,
"do_blacklist": 7,
"do_log": 1,
"effective_rage": 0,
"user_region": "FRAG_UNIT_ID",
"is_valid": "yes",
"groups": [
{
"group_name": "7_start",
"regions": [
{
"table_name": "SIFTER_START",
"table_type": "string",
"table_content": {
"keywords": "http://",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 9,
"service": 2221,
"action": 4,
"do_blacklist": 7,
"do_log": 2,
"effective_rage": 0,
"user_region": "FRAG_UNIT_REOFFSET",
"is_valid": "yes",
"groups": [
{
@@ -236,7 +236,7 @@
"table_name": "SIFTER_START",
"table_type": "string",
"table_content": {
"keywords": "/vmind",
"keywords": "&ts_seg_no=",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
@@ -248,12 +248,12 @@
},
{
"compile_id": 10,
"service": 5222,
"action": 2,
"service": 4122,
"action": 3,
"do_blacklist": 8,
"do_log": 1,
"effective_rage": 0,
"user_region": "FRAG_UNIT_ID",
"user_region": "FRAG_UNIT_ABOFFSET",
"is_valid": "yes",
"groups": [
{
@@ -263,7 +263,7 @@
"table_name": "SIFTER_START",
"table_type": "string",
"table_content": {
"keywords": "/vmind",
"keywords": "range=",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"

12
bin/conf/sifter/urloffset_data.sft
View File

@@ -1,12 +0,0 @@
[INFOR]
sft_id = 3122
[EXTRACT]
#[expect_type][expect_name] = extract_location$extract_method$start$mid$end$
#expect_type and expect_name must be one
#sifter_method = Tag | RawText | Boundary | Reposition
#special character : \r\n = 0x0d0x0a, \x30 = no end , \x0 = null
#expect_name = {1}{MEDIA_ID}, {2}{FRAG_UNIT_ID}, {3}{FRAG_UNIT_ABOFFSET}, {4}{FRAG_UNIT_REOFFSET}, {5}{MEDIA_SIZE}, {6}{MEDIA_NAME}, {7}{MEDIA_CONTENT}
#extract_location = {61}{C2S_URL}, {0xD1}{S2C_CONTENT-DISPOSITION},{0xE1}{C2S_CONTENT},{0xF1}{S2C_CONTENT}
{1}{MEDIA_ID}={61}{C2S_URL}$End$\x0$\x0$f4v$
{4}{FRAG_UNIT_REOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$

2
bin/conf/sifter/youtube_data.sft
View File

@@ -9,4 +9,4 @@ sft_id = 4122
#expect_name = {1}{MEDIA_ID}, {2}{FRAG_UNIT_ID}, {3}{FRAG_UNIT_ABOFFSET}, {4}{FRAG_UNIT_REOFFSET}, {5}{MEDIA_SIZE}, {6}{MEDIA_NAME}, {7}{MEDIA_CONTENT}
#extract_location = {61}{C2S_URL}, {0xD1}{S2C_CONTENT-DISPOSITION},{0xE1}{C2S_CONTENT},{0xF1}{S2C_CONTENT}
{1}{MEDIA_ID}={61}{C2S_URL}$End$\x0$\x0$range=$
{4}{FRAG_UNIT_REOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$
{3}{FRAG_UNIT_ABOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$

BIN
frag_rssb安装使用手册.docx
View File

Binary file not shown.

2
src/AV_interface.h
View File

@@ -36,7 +36,7 @@
#define FILE_IOS 0xA3
#define FILE_ANDRIOD 0xA4
#define FILE_APP 0xA5
#define FILE_FRAG 0xA6 /*<2A><>ƬԤ<C6AC><D4A4>ʶ<EFBFBD><CAB6><EFBFBD><EFBFBD>,<2C><><EFBFBD><EFBFBD>url<72><6C><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ƫ<EFBFBD><C6AB><EFBFBD><EFBFBD><EFBFBD>ģ<EFBFBD><C4A3><EFBFBD><EFBFBD><EFBFBD>iqiyi*/
#define FILE_REQ_FRAG 0xA6 /*<2A><>ƬԤ<C6AC><D4A4>ʶ<EFBFBD><CAB6><EFBFBD><EFBFBD>,<2C><><EFBFBD><EFBFBD>url<72><6C><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ƫ<EFBFBD><C6AB><EFBFBD><EFBFBD><EFBFBD>ģ<EFBFBD><C4A3><EFBFBD><EFBFBD><EFBFBD>iqiyi*/
#define FILE_MAYBE_FRAG 0xA7 /*ǰ<>˻ش<CBBB><D8B4>Ŀ<EFBFBD><C4BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ƭ<EFBFBD><C6AC><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ƶ*/
#define FILE_AV 0x00 /*<2A><>ƬԤ<C6AC><D4A4>ʶ<EFBFBD><CAB6><EFBFBD><EFBFBD>ý<EFBFBD><C3BD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ת<EFBFBD><D7AA>Ϊ0x00*/

10
src/frag_av.c
View File

@@ -751,7 +751,7 @@ void media_stat(media_t* mdi, frag_unit_t* frg_unit)
atomic_inc(&frag_rssb.data_info[RSSB_OSMF_TO_OTHER][TOTAL_PKTS]);
}
break;
case FILE_FRAG:
case FILE_REQ_FRAG:
if(frg_unit->service_id!=0)
{
atomic_inc(&frag_rssb.data_info[RSSB_RECV_FRAG_MEDIA][TOTAL_PKTS]);
@@ -794,7 +794,7 @@ void media_byte_stat(media_t* mdi, frag_unit_t* frg_unit, frag_in_t* frg)
}
break;
case FILE_FRAG:
case FILE_REQ_FRAG:
if(frg_unit!=NULL && frg_unit->service_id!=0)
{
atomic_add(&frag_rssb.data_info[RSSB_RECV_FRAG_MEDIA][TOTAL_BYTES],frg->datalen);
@@ -900,10 +900,10 @@ int media_removal(media_t* mdi, frag_unit_t* frg_unit, frag_in_t* frg, frag_ivi_
frg->seq = mdi->re_offset;
frg->offset_in = frg->offset;
frag_stat = 1;
}
}
else
{
if(FILE_FRAG==mdi->media_type && NULL!=frg_unit)
if(FILE_REQ_FRAG==mdi->media_type && NULL!=frg_unit)
{
frg->seq = frg_unit->re_offset;
frg->offset += frg_unit->ab_offset;
@@ -1298,7 +1298,7 @@ long converge_mediainfo_search_cb(void *data, const uint8_t *key, uint size, voi
int is_frag(uint8_t media_type)
{
if(media_type==FILE_OSMF || media_type==FILE_HLS || media_type==FILE_FRAG || media_type==FILE_MAYBE_FRAG ||media_type==FILE_IOS|| media_type==FILE_ANDRIOD|| media_type==FILE_APP)
if(media_type==FILE_OSMF || media_type==FILE_HLS || media_type==FILE_REQ_FRAG || media_type==FILE_MAYBE_FRAG ||media_type==FILE_IOS|| media_type==FILE_ANDRIOD|| media_type==FILE_APP)
{
return 1;
}

24
src/frag_reassembly.c
View File

@@ -481,7 +481,7 @@ void set_media_service_type(media_t* mdi)
mdi->media_service_type = MEDIA_SERVICE_TYPE_FRAG;
mdi->media_len = 0;
}
else if(FILE_FRAG==mdi->media_type)
else if(FILE_REQ_FRAG==mdi->media_type)
{
mdi->media_service_type = MEDIA_SERVICE_TYPE_FRAG;
}
@@ -709,7 +709,7 @@ void free_frag_unit(void* data)
{
case FILE_HLS:
case FILE_OSMF:
case FILE_FRAG:
case FILE_REQ_FRAG:
case FILE_MAYBE_FRAG:
frag_redis_index_twice(frg_unit);
break;
@@ -1045,15 +1045,15 @@ long media_preproc_cb(void *data, const uint8_t *key, uint size, void *user_arg)
}
/*offset==0, log*/
if(0==frg->offset)
{
if(!FLAG_TEST(mdi->flag, PROG_OFFSET_ZERO))
{
atomic_inc(&g_frag_stat.media_stat[LOG_MEDIA_OFFSET_ZERO]);
FLAG_SET(mdi->flag, PROG_OFFSET_ZERO);
}
create_media_write_to_log(mdi, MEDIA_OFFSET_ZERO, NULL);
}
if(0==frg->offset)
{
if(!FLAG_TEST(mdi->flag, PROG_OFFSET_ZERO))
{
atomic_inc(&g_frag_stat.media_stat[LOG_MEDIA_OFFSET_ZERO]);
FLAG_SET(mdi->flag, PROG_OFFSET_ZERO);
}
create_media_write_to_log(mdi, MEDIA_OFFSET_ZERO, NULL);
}
/*record maxoffset*/
mdi->maxoffset = MAX(frg->offset, mdi->maxoffset);
@@ -1335,7 +1335,7 @@ long get_media(void *data, const uint8_t *key, uint size, void *user_arg)
}
/*<2A><><EFBFBD><EFBFBD>ͳ<EFBFBD><CDB3>for data*/
/*<2A><><EFBFBD><EFBFBD>HLS OSMF<4D><46><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ƬЭ<C6AC><D0AD>media_type<70><65><EFBFBD><EFBFBD>Ϊ<EFBFBD><CEAA>ͳ<EFBFBD><CDB3>Ŀ<EFBFBD><C4BF>mediatype*/
if(mdi->media_type==FILE_FRAG)
if(mdi->media_type==FILE_REQ_FRAG)
{
media_info->media_type = FILE_AV;
media_info->prog_len = 0;

11
src/main.c
View File

@@ -42,9 +42,9 @@ const char* frag_rssb_version = "2018-08-13T09:00:00";
const char* frag_rssb_version_time = "2018-08-13T09:00:00";
const char* frag_rssb_version_des = "MESA@iie rssb_maskey";
int FRAG_RSSB_VERSION_1_0_20181211 = 0;
const char* frag_rssb_version_time_in = "2018-12-11";
const char* frag_rssb_version_des_in = "resp_checkresult_search_media_cb add K_PROJECT";
int FRAG_RSSB_VERSION_1_0_20190121 = 0;
const char* frag_rssb_version_time_in = "2019-01-21";
const char* frag_rssb_version_des_in = "support youtube";
void frag_rssb_history()
{
//2015.11.15 v1.0 create the project
@@ -218,7 +218,7 @@ void frag_rssb_history()
//2018.09.13 v4.0 //1.frag removal
//2018.09.20 v4.0//1 voip_fulllog add voice_dir opt
//2018.09.27 v4.0 //1. hard balance
//2018.10.08 v4.0//1.add send_voip_full_json_log for K_PROJECT;2.voip_fulllog and voip_surveylog add voice_dir and cap_ip opt
//2018.10.08 v4.0//1.add send_voip_full_json_log for K_PROJECT;2.voip_fulllog and voip_surveylog add voice_dir and cap_ip opt
//2018.10.22 v4.0//alter g_sip_opt_type and add enum sip_opt_index
//2018.11.02 v4.0//add timer_on
//2018.11.05 v4.0//alter voip save_media frg_len>headerlen
@@ -226,7 +226,8 @@ void frag_rssb_history()
//2018.12.07 v4.0//unixsocket set bufsize timeout in configure
//2018.12.10 v4.0//1.recv_msg_fd not unix 2.MsgPort
//2018.12.11 v4.0//1.file mediatype
//2018.12.11 v4.0//resp_checkresult_search_media_cb add K_PROJECT,and delete send_json_log
//2018.12.11 v4.0//resp_checkresult_search_media_cb add K_PROJECT,and delete send_json_log
//2019.01.21 v4.0//support youtube; update req_frag and template
}
frag_rssb_parameter_t g_frag_run;