gfwleak/av-frag-rssb
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update template youtube.sft

Browse Source
This commit is contained in:
lishu
2018-11-12 11:19:34 +08:00
parent 256c0223b6
commit 8e14263ef6
8 changed files with 42 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
bin/conf/frag_reassembly.conf
View File

@@ -3,10 +3,7 @@ RedisClusterSwitch=1
RedisTimeout=5
#cluster mode
RedisBrokers=10.0.6.192:9012
RedisNet=192
#HIRCLUSTER_FLAG_UNIQIP_FRAG_ONE=0x100=256,HIRCLUSTER_FLAG_UNIQIP_FRAG_TWO=0x200=512,HIRCLUSTER_FLAG_UNIQIP_FRAG_THREE=0x400=1024,HIRCLUSTER_FLAG_UNIQIP_FRAG_FOUR=0x800=2048
RedisUniqIPFlag=2048
RedisBrokers=192.168.10.205:9012
#no cluster mode
RedisIP=
@@ -21,11 +18,11 @@ ConvergeHashExpireTime=300
[LOG]
#frag reassembly log <20><><EFBFBD><EFBFBD>ƴ<EFBFBD>ӵ<EFBFBD><D3B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>־
FragReassemblyLogLevel=10
FragReassemblyLogLevel=30
FragReassemblyLogName=./frag_reassembly.log
#media log
MediaLogLevel=10
MediaLogLevel=30
MediaLogName=./frag_reassembly_media.log
#stat log

48
bin/conf/main.conf
View File

@@ -1,7 +1,7 @@
#--------------------------------------------------------------------NETWORK
[NETWORK]
LocalIP=em1
FwdIP=em1
LocalIP=ens33
FwdIP=ens33
#recv bizman param, recv data from sapp
BizmanAckSmoothTime = 10
@@ -22,9 +22,9 @@ UnixSocketSendDestAddr=/home/mcger/un_recv;/home/audiorecognition/aufilter/un_re
UnixSocketRecvAddr=/home/mesasoft/frag_rssb/un_recv
#udp socket:send frag
UdpSendIP=127.0.0.1;
UdpSendIPNum=1
UdpSendPort=33082
UdpSendIP=127.0.0.1;127.0.0.1;
UdpSendIPNum=2
UdpSendPort=33082;33083
#<23><><EFBFBD><EFBFBD><EFBFBD>ڴ<EFBFBD><DAB4><EFBFBD><EFBFBD><EFBFBD>
USM_ON_FLAG=0
@@ -53,19 +53,19 @@ WhiteListPort=22085
CPZTpye=0
ThreadNum=1
BizmanQueueMode=0
BizmanQueueMode=1
BizmanQueueMaxnum=2000000
AsmisSwitch=1
AsmisSwitch=0
#<23><>ǰ<EFBFBD>˷<EFBFBD><CBB7>ͽ<EFBFBD>Ŀȷ<C4BF><C8B7>
AckSwitch=0
#IVIȥ<49>ؿ<EFBFBD><D8BF><EFBFBD>
IVISwitch=0
IVISwitch=1
#<23><><EFBFBD>ͽӿ<CDBD><D3BF>Ƿ<EFBFBD><C7B7><EFBFBD><EFBFBD><EFBFBD>AV_sendback
AVSendbackSwitch=1
AVSendbackSwitch=0
#index<65>ڶ<EFBFBD><DAB6>η<EFBFBD><CEB7><EFBFBD>ʱ<EFBFBD><CAB1>
IndexQueryTime=10
@@ -78,6 +78,9 @@ RenewTimeMin=600
RenewTimeStep=6
#<23><>ĿHASH<53><48>
#MediaHashSize=8
#MediaHashElemNum=16
#MediaHashExpireTime=120
MediaHashSize=65536
MediaHashElemNum=1048576
MediaHashExpireTime=300
@@ -97,16 +100,16 @@ MonitorFileRootDir=/home/yspdata/qd_monitor/
MonitorFileDay=1
#special media type, send windows system
ForwardSpecialMediaSwitch=1
SpecialMediaType=0x68
ForwardSpecialMediaSwitch=0
SpecialMediaType=0x63
#AV record switch
AVRecordFileSwitch=1
AVRecordFileSwitch=0
AVRecordFileRootDir=./AVrecord/
AVRecordFileMaxNum=100000
#ģ<><C4A3>hashժҪ<D5AA><D2AA><EFBFBD>㿪<EFBFBD><E3BFAA>
FuzzyDigestSwitch=1
FuzzyDigestSwitch=0
#Ԫ<><D4AA>Ϣѡ<CFA2><D1A1>capIP<49>Ƿ<EFBFBD><C7B7>޸<EFBFBD> 1<><31><EFBFBD><EFBFBD>Ϊ<EFBFBD><CEAA><EFBFBD><EFBFBD>capIP, 0:ѡ<><D1A1>Ϊǰ<CEAA>˻<EFBFBD>CapIP
ModifyCapIPSwitch=1
@@ -124,7 +127,7 @@ DedupThriftTime=5
DedupHtableArg=0
#<23><><EFBFBD>ض<EFBFBD>Դҵ<D4B4>񿪹<EFBFBD>
AVDedupSwitch=1
AVDedupSwitch=0
AVDedupInvalid=0
#td=(URL+ServerIP...)+data(<=65536)
DedupTdDataSize=65536
@@ -146,7 +149,7 @@ MediaJSONSwitch=1
#==============================================================VOIP
[VOIP]
VOIPFilterSwitch=1
VOIPFilterSwitch=0
#VOIP survey log <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>־
VOIPSurveyLogIP=
@@ -159,7 +162,7 @@ VOIPFullLogIPNum=
VOIPFullLogPort=
#voip log
VOIPLogLevel= 30
VOIPLogLevel= 20
VOIPLogPath= ./log/voip.log
#==============================================================APP
@@ -202,13 +205,13 @@ MediaExpireLogPath= ./log/media_expire.log
#--------------------------------------------------------------------DEBUG
[DEBUG]
/*<2A><>Ŀ<EFBFBD><C4BF><EFBFBD>б<EFBFBD><D0B1>ش洢*/
FileStoreSwitch=0
FileStoreSwitch=1
FileStorePath=./log/file/
/*<2A><><EFBFBD>н<EFBFBD>Ŀ<EFBFBD><C4BF><EFBFBD><EFBFBD><EFBFBD>б<EFBFBD><D0B1>ظ<EFBFBD><D8B8>ٻ<EFBFBD>д*/
AllHitMonitorSwitch=2
AllHitMonitorCompleteRate=90
AllHitMonitorFilenameType=1
AllHitMonitorSwitch=0
AllHitMonitorCompleteRate=0
AllHitMonitorFilenameType=0
#1: (reoffset-1)*content-length 0: accoffset
HlsAboffsetInMode=1
@@ -221,7 +224,7 @@ JSONLocalSwitch=1
[WLB]
wlb_on=1
wlb_on=0
wlb_topic=AV
wlb_group_name=xx
user_tag=xx
@@ -232,5 +235,4 @@ capacity=32
cost=32
wlb_report_interval=10
bfd_recv_port=
bfd_send_port=
bfd_recv_port=33082

57
bin/conf/sifter/frag_reassembly.conf
View File

@@ -1,57 +0,0 @@
[NETWORK]
RedisBrokers=10.0.6.80:9001,10.0.6.80:9002,10.0.6.80:9003,10.0.6.80:9004,10.0.6.80:9005
RedisTimeout=5
[SYSTEM]
ConvergeHashSize=65536
ConvergeHashElemNum=1048576
ConvergeHashExpireTime=120
#same with protocol in frag_monitor
ProtocolNetdisk=17
ProtocolWebmail=18
#webmail param================================start
WebmailHashSize=65536
WebmailHashElemNum=1048576
WebmailHashExpireTime=120
StoreidAttachHashSize=65536
StoreidAttachHashElemNum=1048576
StoreidAttachHashExpireTime=120
StoreidMidHashSize=65536
StoreidMidHashElemNum=1048576
StoreidMidHashExpireTime=120
#0, not open iterate thread to output
WebmailIterateTimeout=60
WebmailAttachIterateTimeout=60
WebmailService=123,163,sina,sohu,qq
WebmailAction=send,recv
#webmail param================================end
[LOG]
#frag reassembly log <20><><EFBFBD><EFBFBD>ƴ<EFBFBD>ӵ<EFBFBD><D3B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>־
FragReassemblyLogLevel=30
FragReassemblyLogName=./frag_reassembly.log
#media log
MediaLogLevel=30
MediaLogName=./frag_reassembly_media.log
#stat log
StatInterval=5
StatFile=./frag_reassembly_stat.log
SysinfoInterval=5
SysinfoFile=./frag_reassembly_sysinfo.log
#webmail log
WebmailLogLevel=10
WebmailLogName=./webmail.log
#Maat
MaatStatSwitch=0
MaatPerfSwitch=0
#MaatStatFile=./maat_stat.log

147
bin/conf/sifter/main.conf
View File

@@ -1,147 +0,0 @@
[NETWORK]
LocalIP=em1
#recv bizman param, recv data from sapp
BizmanAckSmoothTime = 10
BizmanAckAccumulateTime=10
BizmanAckAccumulateNum=5
BizmanPort=22082
#sapp recv resp msg port
BizmanAckPort=22084
#msg port
MsgPort=22080
#unix-domain socket file path <20><>send frag
UnixSocketSendSrcAddr=/home/lishu/frag/frag_rssb/maskey/bin/un_send
UnixSocketSendDestAddr=/home/lishu/frag/frag_rssb/maskey/bin/data_recv
#udp socket recv port <20><>recv response msg
UnixSocketRecvAddr=/home/lishu/frag/frag_rssb/maskey/bin/un_recv
#usm
USM_ON_FLAG=1
USM_SHM_KEY=1635151988
#USM_SHM_KEY=12346
USM_SHM_SIZE=100000
USM_READER_CNT=3
USM_SMOOTH_TIME=1000000
USM_Q_SIZE=1048576
USM_READER_PATH=/home/mcger/un_recv;/home/audiorecognition/aufilter/un_recv;/home/mesasoft/un_test/un_recv
USM_LOG_LEVEL=40
USM_LOG_PATH=./log/usm_writer0;./log/usm_writer1;./log/usm_writer2
#special media type, send windows system , sep: ";" : IP1+PORT1/PORT2 or IP2+PORT1/PORT2
SpecialMediaWindowsIP=10.0.6.5;10.0.6.1;
SpecialMediaWindowsIPNum=2
SpecialMediaWindowsPort=44096;44098
#white list IP and PORT : IP1+PORT and IP2+PORT
WhiteListIP = 10.52.178.15;
WhiteListIPNum=1
WhiteListPort=22085
#av_dedup service <20><><EFBFBD><EFBFBD>ҵ<EFBFBD><D2B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ϣ
ThriftIP=127.0.0.1;
ThriftIPNum=1
ThriftPort=9090
DedupBalanceNum=1000
DedupQueueNum=100000
DedupStatInterval=60
DedupThriftNum=5
DedupHtableArg=0
#JSON to kafka <20><>ý<EFBFBD><C3BD><EFBFBD><EFBFBD>Ϣ<EFBFBD>ش<EFBFBD>ҵ<EFBFBD><D2B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
MediaJSONKafkaBrokers=
[SYSTEM]
ThreadNum=4
BizmanQueueMode=0
#media renew time
RenewTimeMax=7200
RenewTimeMin=600
RenewTimeStep=6
#<23><>ĿHASH<53><48>
MediaHashSize=65536
MediaHashElemNum=1048576
MediaHashExpireTime=2
#<23><><EFBFBD>ٻ<EFBFBD>дhash<73>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD>DumpfileHashElemNum<75><6D><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>IJ<EFBFBD><C4B2><EFBFBD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD>Ŀ
DumpfileHashSize=1024
DumpfileHashElemNum=512
DumpfileHahsExpireTime=6
#monitor service
MonitorHashSize=65536
MonitorHashElemNum=1048576
MonitorHahsExpireTime=120
MonitorFileSwitch=1
MonitorFileRootDir=/home/yspdata/qd_monitor/
MonitorFileDay=1
#special media type, send windows system
ForwardSpecialMediaSwitch=1
SpecialMediaType=0x68
#AV record switch
AVRecordFileSwitch=0
AVRecordFileRootDir=./AVrecord/
AVRecordFileMaxNum=100000
#<23><><EFBFBD><EFBFBD>ҵ<EFBFBD>񿪹<EFBFBD>
AVDedupSwitch=0
#td=(URL+ServerIP...)+data(<=65536)
DedupTdDataSize=0
#media info send to kfaka <20><>ý<EFBFBD><C3BD>JSON<4F><4E>Ϣ<EFBFBD>ش<EFBFBD>KAFKA<4B><41><EFBFBD><EFBFBD>
MediaJSONSwitch=0
#<23>ֻ<EFBFBD>app<70><70><EFBFBD><EFBFBD>
AppSwitch=0
[LOG]
#field_stat
StatCycle=2
StatFile=./log/rssb_stat.log
SysinfoCycle=2
SysinfoFile=./log/rssb_sysinfo.log
#log in general
LogLevel= 30
LogPath= ./log/runtime.log
#frag msg log
FragLogLevel= 20
FragLogPath= ./log/frag.log
#av_dedup log
DedupLogLevel= 10
DedupLogPath= ./log/dedup.log
#resp msg log
RespLogPath= ./log/survey.log
#media create log
MediaCreateLogPath= ./log/media_create.log
#media expire log
MediaExpireLogPath= ./log/media_expire.log
[DEBUG]
/*<2A><><EFBFBD><EFBFBD>mid_trace.conf<6E>Ľ<EFBFBD>Ŀ<EFBFBD><C4BF><EFBFBD>б<EFBFBD><D0B1>ش洢*/
FileStoreSwitch=0
FileStorePath=./log/file/
/*<2A><><EFBFBD>н<EFBFBD>Ŀ<EFBFBD><C4BF><EFBFBD><EFBFBD><EFBFBD>б<EFBFBD><D0B1>ظ<EFBFBD><D8B8>ٻ<EFBFBD>д*/
AllHitMonitorSwitch=0
#1: (reoffset-1)*content-length 0: accoffset
HlsAboffsetInMode=1
/*trace log path*/
TracePath=./log/

1
bin/conf/sifter/mid_trace.conf
View File

@@ -1 +0,0 @@
#mid or pid

11
bin/conf/sifter/table_info.conf
View File

@@ -1,11 +0,0 @@
#each collumn seperate with '\t'
#id (0~65535(
#name string
#type one of ip,expr,compile or plugin
#src_charset one of GBK,BIG5,UNICODE,UTF8
#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/'
#do_merege yes or no
#id name type src_charset dst_charset do_merge
1 SIFTER_CONFIG_FULL compile GBK GBK no
2 SIFTER_GROUP group GBK GBK no
3 SIFTER_START expr GBK GBK yes

12
bin/conf/sifter/youtube_data.sft Normal file
View File

@@ -0,0 +1,12 @@
[INFOR]
sft_id = 4122
[EXTRACT]
#[expect_type][expect_name] = extract_location$extract_method$start$mid$end$
#expect_type and expect_name must be one
#sifter_method = Tag | RawText | Boundary | Reposition
#special character : \r\n = 0x0d0x0a, \x30 = no end , \x0 = null
#expect_name = {1}{MEDIA_ID}, {2}{FRAG_UNIT_ID}, {3}{FRAG_UNIT_ABOFFSET}, {4}{FRAG_UNIT_REOFFSET}, {5}{MEDIA_SIZE}, {6}{MEDIA_NAME}, {7}{MEDIA_CONTENT}
#extract_location = {61}{C2S_URL}, {0xD1}{S2C_CONTENT-DISPOSITION},{0xE1}{C2S_CONTENT},{0xF1}{S2C_CONTENT}
{1}{MEDIA_ID}={61}{C2S_URL}$End$\x0$\x0$range=$
{4}{FRAG_UNIT_REOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$

4
src/main.h
View File

@@ -62,13 +62,13 @@ typedef struct frag_rssb_parameter_s
MESA_htable_handle media_hash; //media_hash
MESA_timer_t* index_query_timer[MAX_THREAD_NUM];
char index_query_timer_on[MAX_THREAD_NUM];
char index_query_timer_on[MAX_THREAD_NUM];
/*<2A><>Դ<EFBFBD><D4B4><EFBFBD><EFBFBD>ҵ<EFBFBD><D2B5>*/
void* dedup_hd; //av_dedup_handle
void* dedup_logger; //av_dedup log
MESA_timer_t* multisrc_timer[MAX_THREAD_NUM];
char multisrc_timer_on[MAX_THREAD_NUM];
char multisrc_timer_on[MAX_THREAD_NUM];
KafkaProducer* kafka_producer; //kafka