From 8e14263ef6681f056fcb99eb9021cc59bf4170c7 Mon Sep 17 00:00:00 2001 From: lishu Date: Mon, 12 Nov 2018 11:19:34 +0800 Subject: [PATCH] update template youtube.sft --- bin/conf/frag_reassembly.conf | 9 +- bin/conf/main.conf | 48 ++++----- bin/conf/sifter/frag_reassembly.conf | 57 ----------- bin/conf/sifter/main.conf | 147 --------------------------- bin/conf/sifter/mid_trace.conf | 1 - bin/conf/sifter/table_info.conf | 11 -- bin/conf/sifter/youtube_data.sft | 12 +++ src/main.h | 4 +- 8 files changed, 42 insertions(+), 247 deletions(-) delete mode 100644 bin/conf/sifter/frag_reassembly.conf delete mode 100644 bin/conf/sifter/main.conf delete mode 100644 bin/conf/sifter/mid_trace.conf delete mode 100644 bin/conf/sifter/table_info.conf create mode 100644 bin/conf/sifter/youtube_data.sft diff --git a/bin/conf/frag_reassembly.conf b/bin/conf/frag_reassembly.conf index f0f87db..7ccd557 100644 --- a/bin/conf/frag_reassembly.conf +++ b/bin/conf/frag_reassembly.conf @@ -3,10 +3,7 @@ RedisClusterSwitch=1 RedisTimeout=5 #cluster mode -RedisBrokers=10.0.6.192:9012 -RedisNet=192 -#HIRCLUSTER_FLAG_UNIQIP_FRAG_ONE=0x100=256,HIRCLUSTER_FLAG_UNIQIP_FRAG_TWO=0x200=512,HIRCLUSTER_FLAG_UNIQIP_FRAG_THREE=0x400=1024,HIRCLUSTER_FLAG_UNIQIP_FRAG_FOUR=0x800=2048 -RedisUniqIPFlag=2048 +RedisBrokers=192.168.10.205:9012 #no cluster mode RedisIP= @@ -21,11 +18,11 @@ ConvergeHashExpireTime=300 [LOG] #frag reassembly log 整形拼接的所有日志 -FragReassemblyLogLevel=10 +FragReassemblyLogLevel=30 FragReassemblyLogName=./frag_reassembly.log #media log -MediaLogLevel=10 +MediaLogLevel=30 MediaLogName=./frag_reassembly_media.log #stat log diff --git a/bin/conf/main.conf b/bin/conf/main.conf index 6906ca9..b6a5258 100644 --- a/bin/conf/main.conf +++ b/bin/conf/main.conf @@ -1,7 +1,7 @@ #--------------------------------------------------------------------NETWORK [NETWORK] -LocalIP=em1 -FwdIP=em1 +LocalIP=ens33 +FwdIP=ens33 #recv bizman param, recv data from sapp BizmanAckSmoothTime = 10 @@ -22,9 +22,9 @@ UnixSocketSendDestAddr=/home/mcger/un_recv;/home/audiorecognition/aufilter/un_re UnixSocketRecvAddr=/home/mesasoft/frag_rssb/un_recv #udp socket:send frag -UdpSendIP=127.0.0.1; -UdpSendIPNum=1 -UdpSendPort=33082 +UdpSendIP=127.0.0.1;127.0.0.1; +UdpSendIPNum=2 +UdpSendPort=33082;33083 #共享内存配置 USM_ON_FLAG=0 @@ -53,19 +53,19 @@ WhiteListPort=22085 CPZTpye=0 ThreadNum=1 -BizmanQueueMode=0 +BizmanQueueMode=1 BizmanQueueMaxnum=2000000 -AsmisSwitch=1 +AsmisSwitch=0 #向前端发送节目确认 AckSwitch=0 #IVI去重开关 -IVISwitch=0 +IVISwitch=1 #发送接口是否启用AV_sendback -AVSendbackSwitch=1 +AVSendbackSwitch=0 #index第二次反查时间 IndexQueryTime=10 @@ -78,6 +78,9 @@ RenewTimeMin=600 RenewTimeStep=6 #节目HASH表 +#MediaHashSize=8 +#MediaHashElemNum=16 +#MediaHashExpireTime=120 MediaHashSize=65536 MediaHashElemNum=1048576 MediaHashExpireTime=300 @@ -97,16 +100,16 @@ MonitorFileRootDir=/home/yspdata/qd_monitor/ MonitorFileDay=1 #special media type, send windows system -ForwardSpecialMediaSwitch=1 -SpecialMediaType=0x68 +ForwardSpecialMediaSwitch=0 +SpecialMediaType=0x63 #AV record switch -AVRecordFileSwitch=1 +AVRecordFileSwitch=0 AVRecordFileRootDir=./AVrecord/ AVRecordFileMaxNum=100000 #模糊hash摘要计算开关 -FuzzyDigestSwitch=1 +FuzzyDigestSwitch=0 #元信息选项capIP是否修改 1：改为本地capIP, 0:选项为前端机CapIP ModifyCapIPSwitch=1 @@ -124,7 +127,7 @@ DedupThriftTime=5 DedupHtableArg=0 #降载多源业务开关 -AVDedupSwitch=1 +AVDedupSwitch=0 AVDedupInvalid=0 #td=(URL+ServerIP...)+data(<=65536) DedupTdDataSize=65536 @@ -146,7 +149,7 @@ MediaJSONSwitch=1 #==============================================================VOIP [VOIP] -VOIPFilterSwitch=1 +VOIPFilterSwitch=0 #VOIP survey log 监测日志 VOIPSurveyLogIP= @@ -159,7 +162,7 @@ VOIPFullLogIPNum= VOIPFullLogPort= #voip log -VOIPLogLevel= 30 +VOIPLogLevel= 20 VOIPLogPath= ./log/voip.log #==============================================================APP @@ -202,13 +205,13 @@ MediaExpireLogPath= ./log/media_expire.log #--------------------------------------------------------------------DEBUG [DEBUG] /*节目进行本地存储*/ -FileStoreSwitch=0 +FileStoreSwitch=1 FileStorePath=./log/file/ /*所有节目都进行本地高速回写*/ -AllHitMonitorSwitch=2 -AllHitMonitorCompleteRate=90 -AllHitMonitorFilenameType=1 +AllHitMonitorSwitch=0 +AllHitMonitorCompleteRate=0 +AllHitMonitorFilenameType=0 #1: (reoffset-1)*content-length 0: accoffset HlsAboffsetInMode=1 @@ -221,7 +224,7 @@ JSONLocalSwitch=1 [WLB] -wlb_on=1 +wlb_on=0 wlb_topic=AV wlb_group_name=xx user_tag=xx @@ -232,5 +235,4 @@ capacity=32 cost=32 wlb_report_interval=10 -bfd_recv_port= -bfd_send_port= \ No newline at end of file +bfd_recv_port=33082 diff --git a/bin/conf/sifter/frag_reassembly.conf b/bin/conf/sifter/frag_reassembly.conf deleted file mode 100644 index 33df3b6..0000000 --- a/bin/conf/sifter/frag_reassembly.conf +++ /dev/null @@ -1,57 +0,0 @@ -[NETWORK] -RedisBrokers=10.0.6.80:9001,10.0.6.80:9002,10.0.6.80:9003,10.0.6.80:9004,10.0.6.80:9005 -RedisTimeout=5 - -[SYSTEM] -ConvergeHashSize=65536 -ConvergeHashElemNum=1048576 -ConvergeHashExpireTime=120 - -#same with protocol in frag_monitor -ProtocolNetdisk=17 -ProtocolWebmail=18 - -#webmail param================================start -WebmailHashSize=65536 -WebmailHashElemNum=1048576 -WebmailHashExpireTime=120 - -StoreidAttachHashSize=65536 -StoreidAttachHashElemNum=1048576 -StoreidAttachHashExpireTime=120 - -StoreidMidHashSize=65536 -StoreidMidHashElemNum=1048576 -StoreidMidHashExpireTime=120 - -#0, not open iterate thread to output -WebmailIterateTimeout=60 -WebmailAttachIterateTimeout=60 - -WebmailService=123,163,sina,sohu,qq -WebmailAction=send,recv -#webmail param================================end - -[LOG] -#frag reassembly log 整形拼接的所有日志 -FragReassemblyLogLevel=30 -FragReassemblyLogName=./frag_reassembly.log - -#media log -MediaLogLevel=30 -MediaLogName=./frag_reassembly_media.log - -#stat log -StatInterval=5 -StatFile=./frag_reassembly_stat.log -SysinfoInterval=5 -SysinfoFile=./frag_reassembly_sysinfo.log - -#webmail log -WebmailLogLevel=10 -WebmailLogName=./webmail.log - -#Maat -MaatStatSwitch=0 -MaatPerfSwitch=0 -#MaatStatFile=./maat_stat.log diff --git a/bin/conf/sifter/main.conf b/bin/conf/sifter/main.conf deleted file mode 100644 index 8d3b761..0000000 --- a/bin/conf/sifter/main.conf +++ /dev/null @@ -1,147 +0,0 @@ -[NETWORK] -LocalIP=em1 - -#recv bizman param, recv data from sapp -BizmanAckSmoothTime = 10 -BizmanAckAccumulateTime=10 -BizmanAckAccumulateNum=5 -BizmanPort=22082 - -#sapp recv resp msg port -BizmanAckPort=22084 - -#msg port -MsgPort=22080 - -#unix-domain socket file path ：send frag -UnixSocketSendSrcAddr=/home/lishu/frag/frag_rssb/maskey/bin/un_send -UnixSocketSendDestAddr=/home/lishu/frag/frag_rssb/maskey/bin/data_recv - -#udp socket recv port ：recv response msg -UnixSocketRecvAddr=/home/lishu/frag/frag_rssb/maskey/bin/un_recv - -#usm -USM_ON_FLAG=1 -USM_SHM_KEY=1635151988 -#USM_SHM_KEY=12346 -USM_SHM_SIZE=100000 -USM_READER_CNT=3 -USM_SMOOTH_TIME=1000000 -USM_Q_SIZE=1048576 -USM_READER_PATH=/home/mcger/un_recv;/home/audiorecognition/aufilter/un_recv;/home/mesasoft/un_test/un_recv -USM_LOG_LEVEL=40 -USM_LOG_PATH=./log/usm_writer0;./log/usm_writer1;./log/usm_writer2 - -#special media type, send windows system , sep: ";" : IP1+PORT1/PORT2 or IP2+PORT1/PORT2 -SpecialMediaWindowsIP=10.0.6.5;10.0.6.1; -SpecialMediaWindowsIPNum=2 -SpecialMediaWindowsPort=44096;44098 - -#white list IP and PORT : IP1+PORT and IP2+PORT -WhiteListIP = 10.52.178.15; -WhiteListIPNum=1 -WhiteListPort=22085 - -#av_dedup service 降载业务的网络配置信息 -ThriftIP=127.0.0.1; -ThriftIPNum=1 -ThriftPort=9090 -DedupBalanceNum=1000 -DedupQueueNum=100000 -DedupStatInterval=60 -DedupThriftNum=5 -DedupHtableArg=0 - -#JSON to kafka 跨媒体信息回传业务的网络配置 -MediaJSONKafkaBrokers= - - -[SYSTEM] -ThreadNum=4 -BizmanQueueMode=0 - -#media renew time -RenewTimeMax=7200 -RenewTimeMin=600 -RenewTimeStep=6 - -#节目HASH表 -MediaHashSize=65536 -MediaHashElemNum=1048576 -MediaHashExpireTime=2 - -#高速回写hash文件管理DumpfileHashElemNum就是最大的并发文件数目 -DumpfileHashSize=1024 -DumpfileHashElemNum=512 -DumpfileHahsExpireTime=6 - -#monitor service -MonitorHashSize=65536 -MonitorHashElemNum=1048576 -MonitorHahsExpireTime=120 - -MonitorFileSwitch=1 -MonitorFileRootDir=/home/yspdata/qd_monitor/ -MonitorFileDay=1 - -#special media type, send windows system -ForwardSpecialMediaSwitch=1 -SpecialMediaType=0x68 - -#AV record switch -AVRecordFileSwitch=0 -AVRecordFileRootDir=./AVrecord/ -AVRecordFileMaxNum=100000 - -#降载业务开关 -AVDedupSwitch=0 -#td=(URL+ServerIP...)+data(<=65536) -DedupTdDataSize=0 - -#media info send to kfaka 跨媒体JSON信息回传KAFKA开关 -MediaJSONSwitch=0 - -#手机app开关 -AppSwitch=0 - -[LOG] -#field_stat -StatCycle=2 -StatFile=./log/rssb_stat.log - -SysinfoCycle=2 -SysinfoFile=./log/rssb_sysinfo.log - -#log in general -LogLevel= 30 -LogPath= ./log/runtime.log - -#frag msg log -FragLogLevel= 20 -FragLogPath= ./log/frag.log - -#av_dedup log -DedupLogLevel= 10 -DedupLogPath= ./log/dedup.log - -#resp msg log -RespLogPath= ./log/survey.log - -#media create log -MediaCreateLogPath= ./log/media_create.log - -#media expire log -MediaExpireLogPath= ./log/media_expire.log - -[DEBUG] -/*命中mid_trace.conf的节目进行本地存储*/ -FileStoreSwitch=0 -FileStorePath=./log/file/ - -/*所有节目都进行本地高速回写*/ -AllHitMonitorSwitch=0 -#1: (reoffset-1)*content-length 0: accoffset -HlsAboffsetInMode=1 - -/*trace log path*/ -TracePath=./log/ diff --git a/bin/conf/sifter/mid_trace.conf b/bin/conf/sifter/mid_trace.conf deleted file mode 100644 index 1235696..0000000 --- a/bin/conf/sifter/mid_trace.conf +++ /dev/null @@ -1 +0,0 @@ -#mid or pid diff --git a/bin/conf/sifter/table_info.conf b/bin/conf/sifter/table_info.conf deleted file mode 100644 index ae0c330..0000000 --- a/bin/conf/sifter/table_info.conf +++ /dev/null @@ -1,11 +0,0 @@ -#each collumn seperate with '\t' -#id (0~65535( -#name string -#type one of ip,expr,compile or plugin -#src_charset one of GBK,BIG5,UNICODE,UTF8 -#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/' -#do_merege yes or no -#id name type src_charset dst_charset do_merge -1 SIFTER_CONFIG_FULL compile GBK GBK no -2 SIFTER_GROUP group GBK GBK no -3 SIFTER_START expr GBK GBK yes diff --git a/bin/conf/sifter/youtube_data.sft b/bin/conf/sifter/youtube_data.sft new file mode 100644 index 0000000..518173b --- /dev/null +++ b/bin/conf/sifter/youtube_data.sft @@ -0,0 +1,12 @@ +[INFOR] +sft_id = 4122 + +[EXTRACT] +#[expect_type][expect_name] = extract_location$extract_method$start$mid$end$ +#expect_type and expect_name must be one +#sifter_method = Tag | RawText | Boundary | Reposition +#special character : \r\n = 0x0d0x0a, \x30 = no end , \x0 = null +#expect_name = {1}{MEDIA_ID}, {2}{FRAG_UNIT_ID}, {3}{FRAG_UNIT_ABOFFSET}, {4}{FRAG_UNIT_REOFFSET}, {5}{MEDIA_SIZE}, {6}{MEDIA_NAME}, {7}{MEDIA_CONTENT} +#extract_location = {61}{C2S_URL}, {0xD1}{S2C_CONTENT-DISPOSITION},{0xE1}{C2S_CONTENT},{0xF1}{S2C_CONTENT} +{1}{MEDIA_ID}={61}{C2S_URL}$End$\x0$\x0$range=$ +{4}{FRAG_UNIT_REOFFSET}={61}{C2S_URL}$Tag$range=$\x0$-$ diff --git a/src/main.h b/src/main.h index 93337c0..4e03cae 100644 --- a/src/main.h +++ b/src/main.h @@ -62,13 +62,13 @@ typedef struct frag_rssb_parameter_s MESA_htable_handle media_hash; //media_hash MESA_timer_t* index_query_timer[MAX_THREAD_NUM]; - char index_query_timer_on[MAX_THREAD_NUM]; + char index_query_timer_on[MAX_THREAD_NUM]; /*多源降载业务*/ void* dedup_hd; //av_dedup_handle void* dedup_logger; //av_dedup log MESA_timer_t* multisrc_timer[MAX_THREAD_NUM]; - char multisrc_timer_on[MAX_THREAD_NUM]; + char multisrc_timer_on[MAX_THREAD_NUM]; KafkaProducer* kafka_producer; //kafka