diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bed8336 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# Created by .ignore support plugin (hsz.mobi) +.idea/ +target diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml new file mode 100644 index 0000000..bc8ab3b --- /dev/null +++ b/dependency-reduced-pom.xml @@ -0,0 +1,143 @@ + + + 4.0.0 + org.example + xj-log-etl + 1.0-SNAPSHOT + + + + properties + + **/*.properties + + + + properties + + log4j.properties + + + + + + maven-shade-plugin + 2.4.2 + + + package + + shade + + + + + cn.ac.iie.storm.topology.StreamAggregateTopology + + + META-INF/spring.handlers + + + META-INF/spring.schemas + + + + + + + + org.jetbrains.kotlin + kotlin-maven-plugin + ${kotlin.version} + + + compile + compile + + compile + + + + conntest-compile + test-compile + + test-compile + + + + test-compile + test-compile + + test-compile + + + + + 1.8 + + + + maven-compiler-plugin + 2.3.2 + + + compile + compile + + compile + + + + testCompile + test-compile + + testCompile + + + + + 1.8 + 1.8 + + + + + + + nexus + Team Nexus Repository + http://192.168.40.125:8099/content/groups/public + + + + + always + fail + + maven-ali + http://maven.aliyun.com/nexus/content/groups/public/ + + + + + org.jetbrains.kotlin + kotlin-test + 1.4.30-RC + test + + + kotlin-test-common + org.jetbrains.kotlin + + + + + + 1.0.2 + 1.2.16 + UTF-8 + 1.4.30-RC + 1.0.0 + + + diff --git a/resources/common_app_id_coding.properties b/resources/common_app_id_coding.properties new file mode 100644 index 0000000..5161ca0 --- /dev/null +++ b/resources/common_app_id_coding.properties @@ -0,0 +1 @@ +#common_app_id_conding HTTP=01999 DNS=02999 WEBMAIL=03001 SMTP=03002 POP3=03003 IMAP=03004 PPTP=04001 L2TP=04002 OpenVPN=04003 IPSec=04004 QQ=05001 BlackLight=05002 钉钉=05003 翼聊=05004 Hangouts=05005 个信=05006 微信电话本=05007 飞聊=05008 环聊=05009 有信电话=05010 友约=05011 遇见=05012 Kik=05013 微爱=05014 Tango=05015 敢聊=05016 微信=05017 易信=05018 旺信=05019 Tumblr=05020 Blued=05021 网聊=05022 快传=05023 唱吧=05024 比邻=05025 千牛=05026 派派=05027 找对象=05028 IM+=05029 微聚=05030 BBM=05031 LOFTER=05032 Weico=05033 米聊=05034 YY语音=05035 Skype=05036 Path=05037 网易泡泡=05038 有你=05039 微乐=05040 GO短信=05041 和通讯录=05042 约爱=05043 无秘=05044 Wicker=05045 约会吧=05046 hike messager=05047 摩擦=05048 探探=05049 陌陌=05050 来往点点虫=05051 百度Hi=05052 ZANK=05053 小恩爱=05054 飞信=05055 ChatON=05056 QT语音=05057 爱聊=05058 nice好赞=05059 朋友印象=05060 Googletalk=05061 友加=05062 Keechat=05063 啪啪啪啪音乐圈=05064 微会=05065 Radicall=05066 槽厂=05067 想恋爱=05068 颜值颜值点评=05069 LinkedIn=05070 Telegram=05071 DiDi=05072 ICQ=05073 Zello=05074 NextDoor=05075 Pinterest=05076 叮咚=05077 床上=05078 Nimbuzz=05079 微人脉=05080 陪陪=05081 微密=05082 Vine=05083 Line=05084 between=05085 facebook=05086 Viber=05087 MeetUP=05088 MeetMe=05089 OkCupidDating=05090 碰碰=05091 Diaspora=05092 咚呱=05093 陪聊=05094 Twitter=05095 Kakao=05096 MSN=05097 Voxer=05098 Airetalk=05099 ooVoo=05100 Mico=05101 Banjo=05102 TextNow=05103 抱抱=05104 多多的鱼=05105 个性=05106 闺蜜圈=05107 Instagram=05108 Whatsapp=05109 Talkbox=05110 ChatOn=05111 Coco=05112 Badoo=05113 Match.com=05114 TextFree=05115 Tapatalk=05116 KC网络电话=05117 爱唱=05118 酷我=05119 荔枝FM=05120 铁血=05121 酷友社区=05122 花田交友=05123 红娘网=05124 知乎=05125 百度贴吧=05126 百合婚恋=05127 珍爱网=05128 猫扑=05129 新浪微博=05130 开心网=05131 天际好好约=05132 天涯社区=05133 人人网=05134 世纪佳缘=05135 We heart it=05136 Avocado=05137 Tinder=05138 Swarm=05139 Snapchat=05140 Dating for everyone=05141 Skout=05142 QQ空间=05143 Couple=05144 Happn=05145 FTP=06999 SSL=07999 IP黑=08001 IP监测=08002 IP日志总表=08003 100100=05001 100200=05002 100300=05003 100400=05004 100500=05005 100600=05006 100700=05007 100800=05008 100900=05009 101000=05010 101100=05011 101200=05012 101300=05013 101400=05014 101500=05015 101600=05016 101700=05017 101800=05018 101900=05019 102000=05020 102100=05021 102200=05022 102300=05023 102400=05024 102500=05025 102600=05026 102700=05027 102800=05028 102900=05029 103000=05030 103100=05031 103200=05032 103300=05033 103400=05034 103500=05035 103600=05036 103700=05037 103800=05038 103900=05039 104000=05040 104100=05041 104200=05042 104300=05043 104400=05044 104500=05045 104600=05046 104700=05047 104800=05048 104900=05049 105000=05050 105100=05051 105200=05052 105300=05053 105400=05054 105500=05055 105600=05056 105700=05057 105800=05058 105900=05059 106000=05060 106100=05061 106200=05062 106300=05063 106400=05064 106500=05065 106600=05066 106700=05067 106800=05068 106900=05069 107000=05070 107100=05071 107200=05072 107300=05073 107400=05074 107500=05075 107600=05076 107700=05077 107800=05078 107900=05079 108000=05080 108100=05081 108200=05082 108300=05083 108400=05084 108500=05085 108600=05086 108700=05087 108800=05088 108900=05089 109000=05090 109100=05091 109200=05092 109300=05093 109400=05094 109500=05095 109600=05096 109700=05097 109800=05098 109900=05099 110000=05100 110100=05101 110200=05102 110300=05103 110400=05104 110500=05105 110600=05106 110700=05107 110800=05108 110900=05109 111000=05110 111100=05111 111200=05112 111300=05113 111400=05114 111500=05115 111600=05116 111700=05117 111800=05118 111900=05119 112000=05120 112100=05121 112200=05122 112300=05123 112400=05124 112500=05125 112600=05126 112700=05127 112800=05128 112900=05129 113000=05130 113100=05131 113200=05132 113300=05133 113400=05134 113500=05135 113600=05136 113700=05137 113800=05138 113900=05139 114000=05140 114100=05141 114200=05142 114300=05143 114400=05144 114500=05145 \ No newline at end of file diff --git a/resources/common_app_label_coding.properties b/resources/common_app_label_coding.properties new file mode 100644 index 0000000..f89b09f --- /dev/null +++ b/resources/common_app_label_coding.properties @@ -0,0 +1 @@ +全部=00 HTTP=01 DNS=02 MAIL=03 VPN=04 JSTX=05 FTP=06 SSL=07 IP=08 \ No newline at end of file diff --git a/resources/common_city_coding.properties b/resources/common_city_coding.properties new file mode 100644 index 0000000..a5b79d2 --- /dev/null +++ b/resources/common_city_coding.properties @@ -0,0 +1,292 @@ +China.Hebei.Langfang=131000 +China.Anhui.Ma'anshan=340500 +China.Sichuan.Panzhihua=510400 +China.Guangdong.Foshan=440600 +China.Sichuan.Ziyang=512000 +China.Hunan.Shaoyang=430500 +China.Henan.Luoyang=410300 +China.Sichuan.Ya'an=511800 +China.Gansu.Qingyang=621000 +China.InnerMongoliaAutonomousRegion.Hulunbeier=150700 +China.Shandong.Qingdao=370200 +China.Jiangsu.Lianyungang=320700 +China.Anhui.Wuhu=340200 +China.Heilongjiang.Shuangyashan=230500 +China.Jiangsu.Suzhou=320500 +China.Heilongjiang.Jiamusi=230800 +China.GuangxiZhuangAutonomousRegion.Liuzhou=450200 +China.Anhui.Chuzhou=341100 +China.Fujian.Longyan=350800 +China.Anhui.Huangshan=341000 +China.Fujian.Sanming=350400 +China.Sichuan.Guang'an=511600 +China.Shanxi.Datong=140200 +China.Hebei.Zhangjiakou=130700 +China.Liaoning.Shenyang=210100 +China.Hubei.Suizhou=421300 +China.Jiangsu.Xuzhou=320300 +China.Sichuan.Yibin=511500 +China.Shaanxi.xi'an=610100 +China.Anhui.Chizhou=341700 +China.Guangdong.Zhaoqing=441200 +China.Jiangsu.Changzhou=320400 +China.Fujian.Nanping=350700 +China.Hebei.Tangshan=130200 +China.Jiangsu.Huai'an=320800 +China.Hebei.Qinhuangdao=130300 +China.GuangxiZhuangAutonomousRegion.Yulin=450900 +China.Hunan.Xiangtan=430300 +China.Shandong.Weihai=371000 +China.Guangdong.Zhongshan=442000 +China.Hubei.Huangshi=420200 +China.Henan.Zhengzhou=410100 +China.Hebei.Baoding=130600 +China.Jiangxi.Shang'rao=361100 +China.Liaoning.Panjin=211100 +China.Jilin.Changchun=220100 +China.Shaanxi.Xianyang=610400 +China.Hebei.Cangzhou=130900 +China.Fujian.Zhangzhou=350600 +China.Heilongjiang.Heihe=231100 +China.Hubei.Xianning=421200 +China.Guangdong.Qingyuan=441800 +China.Shandong.Jining=370800 +China.Henan.Xuchang=411000 +China.GuangxiZhuangAutonomousRegion.Fangchenggang=450600 +China.Shaanxi.Baoji=610300 +China.Shandong.Linyi=371300 +China.Sichuan.Dazhou=511700 +China.Yunnan.Dali=532900 +China.Henan.Puyang=410900 +China.Henan.Pingdingshan=410400 +China.Sichuan.Neijiang=511000 +China.Shanxi.Yangquan=140300 +China.Henan.Xinxiang=410700 +China.Guangdong.Yunfu=445300 +China.Hainan.Danzhou=460400 +China.GuangxiZhuangAutonomousRegion.Laibin=451300 +China.InnerMongoliaAutonomousRegion.Baotou=150200 +China.Sichuan.Leshan=511100 +China.Liaoning.Yingkou=210800 +China.Hunan.Chenzhou=431000 +China.Zhejiang.Ningbo=330200 +China.Guangdong.Shanwei=441500 +China.InnerMongoliaAutonomousRegion.Bayannaoer=150800 +China.Anhui.Anqing=340800 +China.Henan.Jiaozuo=410800 +China.Hebei.Chengde=130800 +China.InnerMongoliaAutonomousRegion.Wulanchabu=150900 +China.InnerMongoliaAutonomousRegion.Chifeng=150400 +China.GuangxiZhuangAutonomousRegion.Baise=451000 +China.Guangdong.Jieyang=445200 +China.Jiangsu.Wuxi=320200 +China.Heilongjiang.Ha'erbin=230100 +China.Hubei.Enshi=422800 +China.Guangdong.Meizhou=441400 +China.Yunnan.Lincang=530900 +China.Jiangsu.Yangzhou=321000 +China.Shaanxi.Yulin=610800 +China.Jilin.Baishan=220600 +China.Sichuan.Deyang=510600 +China.Gansu.Jiayuguan=620200 +China.Fujian.Putian=350300 +China.Anhui.Suzhou=341300 +China.Anhui.Bangbu=340300 +China.Shandong.RiZhao=371100 +China.Guangdong.Zhanjiang=440800 +China.Anhui.Xuancheng=341800 +China.Henan.Kaifeng=410200 +China.Shandong.Dezhou=371400 +China.Hebei.Xingtai=130500 +China.Liaoning.Fuxin=210900 +China.Sichuan.Nanchong=511300 +China.Shanxi.Taiyuan=140100 +China.InnerMongoliaAutonomousRegion.Wuhai=150300 +China.Henan.Zhoukou=411600 +China.Anhui.Huaibei=340600 +China.Guizhou.Guiyang=520100 +China.Yunnan.Zhaotong=530600 +China.Gansu.Wuwei=620600 +China.Hubei.Xiangyang=420600 +China.Liaoning.Dalian=210200 +China.Heilongjiang.Jixi=230300 +China.Shanxi.Jincheng=140500 +China.Shandong.Laiwu=371200 +China.Henan.Nanyang=411300 +China.Shandong.Jinan=370100 +China.Gansu.Zhangye=620700 +China.Zhejiang.Huzhou=330500 +China.Jilin.Liaoyuan=220400 +China.Shanxi.Yuncheng=140800 +China.Jilin.Tonghua=220500 +China.Liaoning.Funshun=210400 +China.Hunan.Yongzhou=431100 +China.Guangdong.Shantou=440500 +China.Shaanxi.Tongchuan=610200 +China.NingxiaHuiAutonomousRegion.Yinchuan=640100 +China.Hunan.Hengyang=430400 +China.Gansu.Baiyin=620400 +China.Jilin.Baicheng=220800 +China.Anhui.Haozhou=341600 +China.InnerMongoliaAutonomousRegion.Eerduosi=150600 +China.Jiangxi.Jiujiang=360400 +China.Sichuan.Zunyi=520300 +China.Sichuan.Meishan=511400 +China.Heilongjiang.Hegang=230400 +China.Sichuan.Tongren=520600 +China.GuangxiZhuangAutonomousRegion.Hezhou=451100 +China.Henan.Shangqiu=411400 +China.GuangxiZhuangAutonomousRegion.Beihai=450500 +China.Guangdong.Zhuhai=440400 +China.Jilin.Songyuan=220700 +China.Sichuan.Anshun=520400 +China.Qinghai.Haidong=630200 +China.Shandong.Heze=371700 +China.Hunan.Zhangjiajie=430800 +China.GuangxiZhuangAutonomousRegion.Wuzhou=450400 +China.InnerMongoliaAutonomousRegion.Tongliao=150500 +China.InnerMongoliaAutonomousRegion.Alashan=152900 +China.Yunnan.Pu'er=530800 +China.Guangdong.Shaoguan=440200 +China.Sichuan.Bazhong=511900 +China.Zhejiang.Wenzhou=330300 +China.Sichuan.Guangyuan=510800 +China.Henan.Xinyang=411500 +China.NingxiaHuiAutonomousRegion.Zhongwei=640500 +China.Shaanxi.Ankang=610900 +China.Hubei.Wuhan=420100 +China.Hunan.Yueyang=430600 +China.Yunnan.Qujing=530300 +China.Gansu.Dingxi=621100 +China.Sichuan.Suining=510900 +China.Shanxi.Linfen=141000 +China.Hunan.Yiyang=430900 +China.Hunan.Zhuzhou=430200 +China.Henan.Sanmenxia=411200 +China.Jiangxi.Pingxiang=360300 +China.Zhejiang.Jinhua=330700 +China.Heilongjiang.Qiqiha'er=230200 +China.Shanxi.Shuozhou=140600 +China.Guangdong.Jiangmen=440700 +China.Fujian.Fuzhou=350100 +China.Hunan.Changde=430700 +China.Guangdong.Guangzhou=440100 +China.Guangdong.Chaozhou=445100 +China.InnerMongoliaAutonomousRegion.Xingan=152200 +China.Hubei.Xiaogan=420900 +China.Hubei.Huanggang=421100 +China.Zhejiang.Zhoushan=330900 +China.Hebei.Handan=130400 +China.Shandong.Dongying=370500 +China.Jiangsu.Nantong=320600 +China.Hubei.Xingmen=420800 +China.Shanxi.Xinzhou=140900 +China.Shandong.Tai'an=370900 +China.Shanxi.Changzhi=140400 +China.Jiangsu.Zhenjiang=321100 +China.Jiangxi.Ji'an=360800 +China.Jiangsu.Nanjing=320100 +China.Shandong.Liaocheng=371500 +China.Jiangxi.Fuzhou=361000 +China.Henan.Luohe=411100 +China.Henan.Anyang=410500 +China.Liaoning.Huludao=211400 +China.Sichuan.Liupanshui=520200 +China.Gansu.Longnan=621200 +China.Shaanxi.Yan'an=610600 +China.Heilongjiang.Daqing=230600 +China.Shaanxi.Shangluo=611000 +China.Jiangsu.Yancheng=320900 +China.Liaoning.Dandong=210600 +China.NingxiaHuiAutonomousRegion.Zuishan=640200 +China.Hubei.Ezhou=420700 +China.Liaoning.LiaoYang=211000 +China.Anhui.Tongling=340700 +China.InnerMongoliaAutonomousRegion.Xilinguolei=152500 +China.Shandong.Binzhou=371600 +China.Heilongjiang.Mudanjiang=231000 +China.Sichuan.Bijie=520500 +China.Shanxi.Jinzhong=140700 +China.Zhejiang.Taizhou=331000 +China.Sichuan.Chengdou=510100 +China.Sichuan.Zigong=510300 +China.Hubei.Jingzhou=420100 +China.HeBei.Shijiazhuang=130100 +China.Gansu.Jiuquan=620900 +China.GuangxiZhuangAutonomousRegion.Qinzhou=450700 +China.Yunnan.XishuangbannaDaiAutonomousPrefecture=532800 +China.Guangdong.Dongguan=441900 +China.Jiangsu.Suqian=321300 +China.Jiangxi.Yichun=360900 +China.Liaoning.Tieling=211200 +China.Zhejiang.Shaoxing=330600 +China.GuangxiZhuangAutonomousRegion.Nanning=450100 +China.Sichuan.Mianyang=510700 +China.Yunnan.Yuxi=530400 +China.Tibet.Lhasa=540100 +China.Gansu.Lanzhou=620100 +China.Hainan.Haikou=460100 +China.Hunan.Huaihua=431200 +China.GuangxiZhuangAutonomousRegion.Chongzuo=451400 +China.Zhejiang.Quzhou=330800 +China.Shanxi.Lvliang=141100 +China.Zhejiang.Jiaxing=330400 +China.Guangdong.Shenzhen=440300 +China.Gansu.Pingliang=620800 +China.Hainan.Sansha=460300 +China.Guangdong.Yangjiang=441700 +China.Jiangxi.Nanchang=360100 +China.Liaoning.Jinzhou=210700 +China.Jilin.Jilin=220200 +China.Liaoning.Chaoyang=211300 +China.Fujian.Ningde=350900 +China.Gansu.Tianshui=620500 +China.Yunnan.Baoshan=530500 +China.Hubei.Shiyan=420300 +China.Shandong.Zaozhuang=370400 +China.Hainan.Sanya=460200 +China.Liaoning.Anshan=210300 +China.GuangxiZhuangAutonomousRegion.Guigang=450800 +China.Fujian.Quanzhou=350500 +China.Jilin.Siping=220300 +China.Heilongjiang.Suihua=231200 +China.Hunan.Loudi=431300 +China.Anhui.Hefei=340100 +China.Heilongjiang.Yichun=230700 +China.Qinghai.Xining=630100 +China.Anhui.Fuyang=341200 +China.Anhui.Huainan=340400 +China.Gansu.Jinchang=620300 +China.Guangdong.Heyuan=441600 +China.Zhejiang.Hangzhou=330100 +China.Jiangxi.Yingtan=360600 +China.Guangdong.Maoming=440900 +China.Liaoning.Benxi=210500 +China.NingxiaHuiAutonomousRegion.Wuzhong=640300 +China.Henan.Hebi=410600 +China.Zhejiang.Lishui=331100 +China.Guangdong.Huizhou=441300 +China.Yunnan.Kunming=530100 +China.Yunnan.Lijiang=530700 +China.Shandong.Yantai=370600 +China.Hunan.Changsha=430100 +China.Shandong.Zibo=370300 +China.Heilongjiang.Qitaihe=230900 +China.Anhui.Liu'an=341500 +China.Shaanxi.Hanzhong=610700 +China.Fujian.Xiamen=350200 +China.Shandong.Weifang=370700 +China.Hebei.Hengshui=131100 +China.InnerMongoliaAutonomousRegion.Huhehaote=150100 +China.Jiangxi.Xinyu=360500 +China.Sichuan.Luzhou=510500 +China.GuangxiZhuangAutonomousRegion.Hechi=451200 +China.GuangxiZhuangAutonomousRegion.Guilin=450300 +China.Henan.Zhumadian=411700 +China.Hubei.Yichang=420500 +China.InnerMongoliaAutonomousRegion.Other=150000 +China.Shaanxi.Weinan=610500 +China.Jiangxi.Ganzhou=360700 +China.NingxiaHuiAutonomousRegion.Guyuan=640400 +China.Jiangxi.Jingdezhen=360200 +PrivateIP=000000 \ No newline at end of file diff --git a/resources/common_country_coding.properties b/resources/common_country_coding.properties new file mode 100644 index 0000000..994124e --- /dev/null +++ b/resources/common_country_coding.properties @@ -0,0 +1,252 @@ +UnitedStates=1 +Canada=1 +Russia=7 +Egypt=20 +SouthAfrica=27 +Greece=30 +Netherlands=31 +Belgium=32 +France=33 +Spain=34 +Hungary=36 +EastGermany=37 +Yugoslavia=38 +Italy=39 +Romania=40 +Switzerland=41 +Czechoslovakia=42 +Austria=43 +UnitedKingdom=44 +Denmark=44 +Sweden=46 +Norway=47 +Poland=48 +Germany=49 +Peru=51 +Mexico=52 +Cuba=53 +Argentina=54 +Brazil=55 +Chile=56 +Colombia=57 +Venezuela=58 +Malaysia=60 +Australia=61 +Indonesia=62 +Philippines=63 +newZealand=64 +Singapore=65 +Thailand=66 +Japa=81 +Korea=82 +Vietnam=84 +China=86 +Turkey=90 +India=91 +Pakistan=92 +Afghanistan=93 +SriLanka=94 +Myanmar=95 +Iran=98 +SouthSudan=211 +Morocco=212 +Algeria=213 +Tunisia=216 +Libya=218 +Gambia=220 +Senegal=221 +Mauritania=222 +Mali=223 +Guinea=224 +Coted'Ivoire=225 +BurkinaFaso=226 +Niger=227 +Togo=228 +Benin=229 +Mauritius=230 +Liberia=231 +SierraLeone=232 +Ghana=233 +Nigeria=234 +Chad=235 +CentralAfricanRepublic=236 +Cameroon=237 +CapeVerde=238 +SaoTomeandPrincipe=239 +EquatorialGuinea=240 +Gabon=241 +therepublicofCongo=242 +DemocraticRepublicofCongo=243 +Angola=244 +Guinea-Bissau=245 +DiegoGarcia=246 +AscensionIsland=247 +Seychelles=248 +Sudan=249 +Rwanda=250 +Ethiopia=251 +Somalia=252 +Djibouti=253 +Kenya=254 +Tanzania=255 +Uganda=256 +Burundi=257 +Mozambique=258 +Zanzibar=259 +Zambia=260 +Madagascar=261 +Reunion=262 +Zimbabwe=263 +Namibia=264 +Malawi=265 +Lesotho=266 +Botswana=267 +Swaziland=268 +ComorosandMayotte=269 +St.Helena=290 +Eritrea=291 +Aruba=297 +FaroeIslands=298 +Greenland=299 +Gibraltar=350 +Portugal=351 +Luxembourg=352 +Ireland=353 +Iceland=354 +Albania=355 +Malta=356 +Cyprus=357 +Finland=358 +Bulgaria=359 +Lithuania=370 +Latvia=371 +Estonia=372 +Moldova=373 +Armenia=374 +Belarus=375 +Andorra=376 +Monaco=377 +SanMarino=378 +Vatican=379 +Ukraine=380 +Serbia=381 +Montenegro=382 +Croatia=385 +Slovenia=386 +BosniaandHerzegovina=387 +Europeantelephonenumberspace-ringEuropeservice=388 +Macedonia=389 +CzechRepublic=420 +Slovakia=421 +Liechtenstein=423 +FalklandIslands=500 +Belize=501 +Guatemala=502 +Salvador=503 +Honduras=504 +Nicaragua=505 +CostaRica=506 +Panama=507 +SaintPierreandMiquelon=508 +Haiti=509 +Guadeloupe(includingFrenchSaintMartinandSaintBarthelemy)=590 +Bolivia=591 +Guyana=592 +Ecuador=593 +FrenchGuiana=594 +Paraguay=595 +Martinique=596 +Suriname=597 +Uruguay=598 +NetherlandsAntilles=599 +EastTimor=670 +Australianoverseasterritories:Antarctica,ChristmasIsland,CocosIslands,NorfolkIsland=672 +Brunei=673 +Nauru=674 +PapuaNewGuinea=675 +Tonga=676 +SolomonIslands=677 +Vanuatu=678 +Fiji=679 +Palau=680 +WallisandFutunaIslands=681 +CookIslands=682 +Niue=683 +Samoa=685 +Kiribati,GilbertIslands=686 +NewCaledonia=687 +Tuvalu,EllisIslands=688 +FrenchPolynesia=689 +Tokelau=690 +FederatedStatesofMicronesia=691 +MarshallIslands=692 +NorthKorea=850 +Macao,China=853 +Cambodia=855 +Laos=856 +InternationalMaritimeSatelliteOrganization=870 +GlobalPersonalCommunicationService=878 +Bangladesh=880 +Mobilesatellitesystem=881 +Internet=882 +Maldives=960 +Lebanon=961 +Jordan=962 +Syria=963 +Iraq=964 +Kuwait=965 +SaudiArabia=966 +Yemen=967 +Oman=968 +YemenDemocraticRepublic=969 +Palestine=970 +UnitedArabEmirates=971 +Israel=972 +Bahrain=973 +Qatar=974 +Bhutan=975 +Mongolia=976 +Nepal=977 +Internationalrateservice=979 +InternationalTelecommunicationsPublicCommunicationServiceTest=991 +Tajikistan=992 +Turkmenistan=993 +Azerbaijan=994 +Georgia=995 +Kyrgyzstan=996 +Uzbekistan=998 +Bahamas=1242 +Barbados=1246 +Anguilla=1264 +AntiguaandBarbuda=1268 +BritishVirginIslands=1284 +U.S.VirginIslands=1340 +CaymanIslands=1345 +Bermuda=1441 +Grenada=1473 +TurksandCaicosIslands=1649 +Montserrat=1664 +NorthernMarianaIslands=1670 +Guam=1671 +AmericanSamoa=1684 +SaintLucia=1758 +Dominica=1767 +SaintVincentandheGrenadines=1784 +PuertoRico=1787 +DominicanRepublic=1809 +TrinidadandTobago=1868 +SaintKittsandNevis=1869 +Jamaica=1876 +Curacao=5999 +Macao=820000 +HongKong=810000 +Taiwan=710000 + + + + + + + + + diff --git a/resources/common_jstx_id_coding.properties b/resources/common_jstx_id_coding.properties new file mode 100644 index 0000000..ac60791 --- /dev/null +++ b/resources/common_jstx_id_coding.properties @@ -0,0 +1,194 @@ +100=Unknown_TCP +101=Unknown_UDP +102=Unknown_Other +103=DNS +104=FTP +105=FTPS +106=HTTP +107=HTTPS +108=ICMP +109=IKE +110=MAIL +111=IMAPS +112=IPSEC +113=XMPP +114=L2TP +115=NTP +117=POP3S +118=PPTP +119=QUIC +120=SIP +121=SMB +123=SMTPS +124=SPDY +125=SSH +126=SSL +127=SOCKS +128=TELNET +129=DHCP +130=RADIUS +131=OPENVPN +132=STUN +133=TEREDO +134=DTLS +135=DoH +136=ISAKMP +137=MDNS +138=NETBIOS +139=NETFLOW +140=RDP +141=RTCP +142=RTP +143=SLP +144=SNMP +145=SSDP +146=TFTP +147=BJNP +148=LDAP +149=RTMP +150=RTSP +200=05001 +201=05002 +202=05003 +203=05004 +204=05005 +205=05006 +206=05007 +207=05008 +208=05009 +209=05010 +210=05011 +211=05012 +212=05013 +213=05014 +214=05015 +215=05016 +216=05017 +217=05018 +218=05019 +219=05020 +220=05021 +221=05022 +222=05023 +223=05024 +224=05025 +225=05026 +226=05027 +227=05028 +228=05029 +229=05030 +230=05031 +231=05032 +232=05033 +233=05034 +234=05035 +235=05036 +236=05037 +237=05038 +238=05039 +239=05040 +240=05041 +241=05042 +242=05043 +243=05044 +244=05045 +245=05046 +246=05047 +247=05048 +248=05049 +249=05050 +250=05051 +251=05052 +252=05053 +253=05054 +254=05055 +255=05056 +256=05057 +257=05058 +258=05059 +259=05060 +260=05061 +261=05062 +262=05063 +263=05064 +264=05065 +265=05066 +266=05067 +267=05068 +268=05069 +269=05070 +270=05071 +271=05072 +272=05073 +273=05074 +274=05075 +275=05076 +276=05077 +277=05078 +278=05079 +279=05080 +280=05081 +281=05082 +282=05083 +283=05084 +284=05085 +285=05086 +286=05087 +287=05088 +288=05089 +289=05090 +290=05091 +291=05092 +292=05093 +293=05094 +294=05095 +295=05096 +296=05097 +297=05098 +298=05099 +299=05100 +300=05101 +301=05102 +302=05103 +303=05104 +304=05105 +305=05106 +306=05107 +307=05108 +308=05109 +309=05110 +310=05111 +311=05112 +312=05113 +313=05114 +314=05115 +315=05116 +316=05117 +317=05118 +318=05119 +319=05120 +320=05121 +321=05122 +322=05123 +323=05124 +324=05125 +325=05126 +326=05127 +327=05128 +328=05129 +329=05130 +330=05131 +331=05132 +332=05133 +333=05134 +334=05135 +335=05136 +336=05137 +337=05138 +338=05139 +339=05140 +340=05141 +341=05142 +342=05143 +343=05144 +344=05145 \ No newline at end of file diff --git a/resources/common_province_coding.properties b/resources/common_province_coding.properties new file mode 100644 index 0000000..7244591 --- /dev/null +++ b/resources/common_province_coding.properties @@ -0,0 +1,55 @@ +Beijing=110000 +Tianjin=120000 +Hebei=130000 +Shanxi=140000 +InnerMongoliaAutonomousRegion=150000 +Liaoning=210000 +Jilin=220000 +Heilongjiang=230000 +Shanghai=310000 +Jiangsu=320000 +Zhejiang=330000 +Anhui=340000 +Fujian=350000 +Jiangxi=360000 +Shandong=370000 +Henan=410000 +Hubei=420000 +Hunan=430000 +Guangdong=440000 +GuangxiZhuangAutonomousRegion=450000 +Hainan=460000 +Chongqing=500000 +Sichuan=510000 +Guizhou=520000 +Yunnan=530000 +TibetAutonomousRegion=540000 +Shaanxi=610000 +Gansu=620000 +Qinghai=630000 +NingxiaHuiAutonomousRegion=640000 +Taiwan=710000 +HongKong=810000 +Macao=820000 +Tibet=540000 + +TaipeiCity=710100 +Kaohsiung=710200 +Tainan=710300 +TaichungCity=710400 +Nantou=710600 +Keelung=710700 +Hsinchu=710800 +Chiayi=710900 +NewTaipei=711100 +Yilan=711200 +HsinchuCounty=711300 +Taoyuan=711400 +Miaoli=711500 +Changhua=711700 +ChiayiCounty=711900 +Yunlin=712100 +Pingtung=712400 +Taitung=712500 +Hualien=712600 +PenghuCounty=712700 \ No newline at end of file diff --git a/resources/service_bolt_config.properties b/resources/service_bolt_config.properties new file mode 100644 index 0000000..3c25f95 --- /dev/null +++ b/resources/service_bolt_config.properties @@ -0,0 +1,70 @@ +#Flooding-spout-field + + +#conn-bolt-StreamID +conn.dns.streamid=CONN-DNS-SOURCE +conn.ftp.streamid=CONN-FTP-SOURCE +conn.http.streamid=CONN-HTTP-SOURCE +conn.ip.streamid=CONN-IP-SOURCE +conn.jstx.streamid=CONN-JSTX-SOURCE +conn.mail.streamid=CONN-MAIL-SOURCE +conn.ssl.streamid=CONN-SSL-SOURCE +conn.vpn.streamid=CONN-VPN-SOURCE + + +#conn-ComponentId-spout +kafka.spout=KAFKA-SPOUT +transaction.spout=TRANSACTION-SPOUT +session.spout=SESSION-SPOUT + +#conn-ComponentId-bolt +conn.parse.componentid=CONN-PARSE-COMPONENTID + + +conn.ybinputsize.streamid=CONN-YBInputSize-STREAMID + + + +#safe-spout-field +safe.spout.field=SAFE-SOURCE + + +#GK-SOURCE-STREAMID +gk.ip.streamid=GK-IP-STREAMID +gk.dns.streamid=GK-DNS-STREAMID +gk.http.streamid=GK-HTTP-STREAMID +gk.ssl.streamid=GK-SSL-STREAMID +gk.wygjc.streamid=GK-WYGJC-STREAMID +gk.mail.streamid=GK-MAIL-STREAMID +gk.ftp.streamid=GK-FTP-STREAMID +gk.vpn.streamid=GK-VPN-STREAMID +gk.jstx.streamid=GK-JSTX-STREAMID + + +gk.count.streamid=GK-COUNT-STREAMID + + + +#JC-SOURCE-STREAMID +jc.ip.streamid=JC-IP-STREAMID +jc.dns.streamid=JC-DNS-STREAMID +jc.http.streamid=JC-HTTP-STREAMID +jc.ssl.streamid=JC-SSL-STREAMID +jc.wygjc.streamid=JC-WYGJC-STREAMID +jc.mail.streamid=JC-MAIL-STREAMID +jc.ftp.streamid=JC-FTP-STREAMID +jc.vpn.streamid=JC-VPN-STREAMID +jc.jstx.streamid=JC-JSTX-STREAMID +jc.pic.streamid=JC-PIC-STREAMID +jc.media.streamid=JC-MEDIA-STREAMID +jc.count.streamid=JC-COUNT-STREAMID + + + + + + + + + + diff --git a/resources/service_flow_config.properties b/resources/service_flow_config.properties new file mode 100644 index 0000000..0e98f0a --- /dev/null +++ b/resources/service_flow_config.properties @@ -0,0 +1,202 @@ +#管理kafka地址 +bootstrap.servers=192.168.41.29:9092 +auto.offset.reset=latest + + +#运营商的取值 +yys_num=1 +topology.logo=conn +#通联日志kafka主题 +kafka.topic=test +kafka.topic.transaction=SECURITY-EVENT-COMPLETED-LOG + + +#策略日志kafka主题 +kafka.safe.topic=test + + +#消费者组id-读取topic,存储该spout id的消费offset信息，每次启动前需要更改 +group.id=xj-jc-group2021080413231 + +#kafka消费者每次拉取数据的时间 +kafka.consumer.poll=1000 + +#storm topology workers +topology.workers=1 + +#spout并行度，建议与kafka分区一致 +max.spout.parallelism=1 +spout.parallelism=1 + + + +#ack设置 1启动ack 0不启动ack +topology.num.acks=0 + +#spou睡眠间隔 +topology.spout.sleep.time=1 + + +#通联程序批处理的时间 +topology.tick.tuple.freq.secs=1 +#通联程序没批处理的条数 +batch.inter.num=1 + + + + +parse.parallelism=1 +send.parallelism=1 +ip.send.parallelism=1 +ssl.send.parallelism=1 +http.send.parallelism=1 +count.send.parallelism=1 +unstructured.send.parallelism=1 + + + +#count指标数据时间批次（单位为秒） +count.topology.tick.tuple.freq.secs=30 + +#prometheus-httpserver +prometheus.pushgateway.address=10.111.143.240:9274 + + + +#http-post请求超时时间 +http.post.socketTimeOut=1 + + +#http-post发送设置 +http.connectionRequestTimeout=1000 +http.connectTimeout=1000 +http.socketTimeout=1000 +http.sent.count=1 +http.retry.count=2 +http.heard.content.type=utf-8 +http.hearder.user=yd_xgs_source +http.hearder.password=yd_xgs_source@2021 +http.hearder.format=avro +http.hearder.conection=keep-alilve +#http-post后端接受地址 +http.post.ip=10.111.153.253 +http.post.port=10080 + + + +hos.url=http://10.111.143.200:9098/hos/ +hos.pool.maxconnections=400 +hos.pool.maxerrorretry=3 +hos.pool.requesttimeout=0 +hos.pool.connectiontimeout=60 +hos.pool.connectionttl=-1 +hos.pool.sockettimeout=50 + + + + +s3.connectionRequestTimeout=1000 +s3.connectTimeout=1000 +s3.socketTimeout=1000 +s3.sent.count=10 +s3.hearder.username=yd_xgs_s3 +s3.hearder.password=yd_xgs_s3@2021 +s3.hearder.filetype=pcm +s3.hearder.filetitle=ydtest-file +s3.hearder.isfulltex=false +s3.hearder.contenttype=application/octet-stream + +s3.post.ip=10.111.144.4,10.111.144.5,10.111.144.6 +s3.post.port=8888 + +t.http.rzzb=t-http-rzzb +t.http.jczb=t-http-jczb +t.http.gkzb=t-http-gkzb + +t.mail.rzzb=t-mail-rzzb +t.mail.jczb=t-mail-jczb +t.mail.gkzb=t-mail-gkzb + +t.wygjc.jcrz=t-wygjc-jcrz +t.wygjc.gkrz=t-wygjc-gkrz + +t.pic.jczb=t-pic-jczb +t.media.jczb=t-media-jczb + + + + + + +#通联数据的schema +ftp.rzzb.schema={"type":"record","name":"FTP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +dns.rzzb.schema={"type":"record","name":"DNS_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +http.rzzb.schema={"type":"record","name":"HTTP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +ip.rzzb.schema={"type":"record","name":"IP_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +jstx.rzzb.schema={"type":"record","name":"JSTX_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DZ_ID","type":["string","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"URL_AGENT","type":["string","null"]},{"name":"USER_ID","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"RESERVED","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +ssl.rzzb.schema={"type":"record","name":"SSL_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +vpn.rzzb.schema={"type":"record","name":"VPN_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"URL","type":["int","null"]},{"name":"USERNAME","type":["int","null"]},{"name":"PWD","type":["int","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +mail.rzzb.schema={"type":"record","name":"MAIL_RZZB","fields":[{"name":"BHSJ","type":"long"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"MAIL_SENDER","type":["string","null"]},{"name":"MAIL_RECEVER","type":["string","null"]},{"name":"RECEVER_NUM","type":["int","null"]},{"name":"MAIL_CC","type":["string","null"]},{"name":"CC_NUM","type":["int","null"]},{"name":"MAIL_SUBJECT","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"ACCESSORY","type":["int","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"FJFWS","type":["string","null"]},{"name":"FJFWS_SZD","type":["string","null"]},{"name":"SJFWS","type":["string","null"]},{"name":"SJFWS_SZD","type":["string","null"]},{"name":"XYLX","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} + + +conn.post.dns.topic=t_dns_rzzb +conn.post.ftp.topic=t_ftp_rzzb +conn.post.http.topic=t_http_rzzb +conn.post.ip.topic=t_ip_rzzb +conn.post.jstx.topic=t_jstx_rzzb +conn.post.mail.topic=t_mail_rzzb +conn.post.ssl.topic=t_ssl_rzzb +conn.post.vpn.topic=t_vpn_rzzb + + +gk.post.dns.topic=t_dns_gkzb +gk.post.ftp.topic=t_ftp_gkzb +gk.post.http.topic=t_http_gkzb +gk.post.ip.topic=t_ip_gkzb +gk.post.jstx.topic=t_jstx_gkzb +gk.post.mail.topic=t_mail_gkzb +gk.post.ssl.topic=t_ssl_gkzb +gk.post.vpn.topic=t_vpn_gkzb +gk.post.wygjc.topic=t_wygjc_gkrz + + +jc.post.dns.topic=t_dns_jczb +jc.post.ftp.topic=t_ftp_jczb +jc.post.http.topic=t_http_jczb +jc.post.ip.topic=t_ip_jczb +jc.post.jstx.topic=t_jstx_jczb +jc.post.mail.topic=t_mail_jczb +jc.post.ssl.topic=t_ssl_jczb +jc.post.vpn.topic=t_vpn_jczb +jc.post.wygjc.topic=t_wygjc_jcrz +jc.post.pic.topic=t_pic_jczb +jc.post.media.topic=t_media_jczb + + + + + + +#gk-schema +ip.gkrz.schema={"type":"record","name":"IP_GKRZ","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +dns.gkzb.schema={"type":"record","name":"DNS_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":["int","null"]},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +http.gkzb.schema={"type":"record","name":"HTTP_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +ssl.gkzb.schema={"type":"record","name":"SSL_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +wygjc.gkrz.schema={"type":"record","name":"WYGJC_GKRZ","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"JC_URL","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"SSYQ","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +ftp.gkzb.schema={"type":"record","name":"FTPGJZ_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +vpn.gkzb.schema={"type":"record","name":"VPN_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"URL","type":["string","null"]},{"name":"USERNAME","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +jstx.gkzb.schema={"type":"record","name":"JSTX_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"DZ_ID","type":["string","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"USER_ID","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"RESERVED","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +mail.gkzb.schema={"type":"record","name":"MAIL_GKZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"MAIL_SENDER","type":["string","null"]},{"name":"MAIL_RECEVER","type":["string","null"]},{"name":"RECEVER_NUM","type":["int","null"]},{"name":"MAIL_CC","type":["string","null"]},{"name":"CC_NUM","type":["int","null"]},{"name":"MAIL_SUBJECT","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"ACCESSORY","type":["int","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"FJFWS","type":["string","null"]},{"name":"FJFWS_SZD","type":["string","null"]},{"name":"SJFWS","type":["string","null"]},{"name":"SJFWS_SZD","type":["string","null"]},{"name":"XYLX","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} + +#jc-schema +dns.jczb.schema={"type":"record","name":"DNS_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DNS_NAME","type":["string","null"]},{"name":"DNS_VALUE","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +ftp.jczb.schema={"type":"record","name":"FTPGJZ_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +http.jczb.schema={"type":"record","name":"HTTP_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"COOKIE","type":["string","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +ip.jczb.schema={"type":"record","name":"IP_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +jstx.jczb.schema={"type":"record","name":"JSTX_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"DZ_ID","type":["string","null"]},{"name":"HOST","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"USER_AGENT","type":["string","null"]},{"name":"USER_ID","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"RESERVED","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +ssl.jczb.schema={"type":"record","name":"TDZS_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"ZSMC","type":["string","null"]},{"name":"BFZ","type":["string","null"]},{"name":"SYZ","type":["string","null"]},{"name":"FWQMC","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +vpn.jczb.schema={"type":"record","name":"VPN_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"URL","type":["string","null"]},{"name":"USERNAME","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +wygjz.jczb.schema={"type":"record","name":"WYGJC_JCRZ","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":"int"},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"JC_URL","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"SSYQ","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +pic.jczb.schema={"type":"record","name":"PIC_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"YYZL","type":["int","null"]},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"PIC_NAME","type":["string","null"]},{"name":"PIC_URL","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"PIC_TYPE","type":["string","null"]},{"name":"PIC_CONTENT","type":["bytes","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +media.jczb.schema={"type":"record","name":"MEDIA_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"YYZL","type":["int","null"]},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"MEDIA_NAME","type":["string","null"]},{"name":"MEDIA_URL","type":["string","null"]},{"name":"URL","type":["string","null"]},{"name":"MEDIA_TYPE","type":["string","null"]},{"name":"MEDIA_CONTENT","type":["bytes","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} +mail.jczb.schema={"type":"record","name":"MAILGJZ_JCZB","fields":[{"name":"BHSJ","type":"long"},{"name":"PZID","type":"string"},{"name":"FLID","type":"int"},{"name":"XZID","type":"int"},{"name":"LHID","type":"int"},{"name":"YDY","type":"string"},{"name":"MDDY","type":"string"},{"name":"PZLX","type":"int"},{"name":"GKLB","type":["int","null"]},{"name":"YYZL","type":["int","null"]},{"name":"QDJIP","type":"int"},{"name":"SIP","type":"string"},{"name":"DIP","type":"string"},{"name":"SPORT","type":"int"},{"name":"DPORT","type":"int"},{"name":"PROTOCOL","type":"int"},{"name":"PROTO_ID","type":"string"},{"name":"DETAIL_ID","type":"string"},{"name":"CS_PKTS","type":["int","null"]},{"name":"CS_BYTES","type":["long","null"]},{"name":"SC_PKTS","type":["int","null"]},{"name":"SC_BYTES","type":["long","null"]},{"name":"MAIL_SENDER","type":["string","null"]},{"name":"MAIL_RECEVER","type":["string","null"]},{"name":"RECEVER_NUM","type":["int","null"]},{"name":"MAIL_CC","type":["string","null"]},{"name":"CC_NUM","type":["int","null"]},{"name":"MAIL_SUBJECT","type":["string","null"]},{"name":"CONTENT","type":["string","null"]},{"name":"ACCESSORY","type":["int","null"]},{"name":"FILE_NAME","type":["string","null"]},{"name":"FILE_CONTENT","type":["string","null"]},{"name":"PWD","type":["string","null"]},{"name":"FJFWS","type":["string","null"]},{"name":"FJFWS_SZD","type":["string","null"]},{"name":"SJFWS","type":["string","null"]},{"name":"SJFWS_SZD","type":["string","null"]},{"name":"XYLX","type":["string","null"]},{"name":"GJZ","type":["string","null"]},{"name":"YYS","type":["int","null"]},{"name":"YL1","type":"int"},{"name":"YL2","type":["int","null"]},{"name":"YL3","type":["string","null"]},{"name":"YL4","type":["string","null"]},{"name":"YL5","type":["string","null"]}]} diff --git a/src/main/java/cn/ac/iie/storm/spout/ConnectionKafkaSpout.java b/src/main/java/cn/ac/iie/storm/spout/ConnectionKafkaSpout.java new file mode 100644 index 0000000..e774bd1 --- /dev/null +++ b/src/main/java/cn/ac/iie/storm/spout/ConnectionKafkaSpout.java @@ -0,0 +1,74 @@ +package cn.ac.iie.storm.spout; + +import cn.ac.iie.storm.utils.config.StreamAggregateConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.log4j.Logger; +import org.apache.storm.spout.SpoutOutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichSpout; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; + +import java.util.Collections; +import java.util.Map; +import java.util.Properties; + +/** + * kafkaSpout + */ +public class ConnectionKafkaSpout extends BaseRichSpout { + private static final long serialVersionUID = -3363788553406229592L; + private KafkaConsumer consumer; + private SpoutOutputCollector collector = null; + private TopologyContext context = null; + private final static Logger logger = Logger.getLogger(ConnectionKafkaSpout.class); + + private static Properties createConsumerConfig() { + Properties props = new Properties(); + props.put("bootstrap.servers", StreamAggregateConfig.BOOTSTRAP_SERVERS); + props.put("group.id", StreamAggregateConfig.GROUP_ID); + props.put("session.timeout.ms", "60000"); + props.put("max.poll.records", 3000); + props.put("max.partition.fetch.bytes", 31457280); + props.put("auto.offset.reset", StreamAggregateConfig.AUTO_OFFSET_RESET); + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + logger.info("链接kafka"+StreamAggregateConfig.BOOTSTRAP_SERVERS+"成功"); + return props; + } + + @Override + public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) { + this.collector=spoutOutputCollector; + this.context=topologyContext; + Properties prop = createConsumerConfig(); + this.consumer = new KafkaConsumer<>(prop); + this.consumer.subscribe(Collections.singletonList(StreamAggregateConfig.KAFKA_TOPIC)); + } + @Override + public void close() { + consumer.close(); + } + @Override + public void nextTuple() { + try { + ConsumerRecords records = consumer.poll(StreamAggregateConfig.KAFKA_COMSUMER_POLL); + Thread.sleep(StreamAggregateConfig.TOPOLOGY_SPOUT_SLEEP_TIME); + for (ConsumerRecord record : records) { + this.collector.emit(new Values(record.value())); + } + } catch (Exception e) { + logger.error("KafkaSpout发送消息出现异常!", e); + e.printStackTrace(); + } + } + + + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + outputFieldsDeclarer.declare(new Fields("flooding-source")); + } +} diff --git a/src/main/java/cn/ac/iie/storm/spout/SessionKafkaSpout.java b/src/main/java/cn/ac/iie/storm/spout/SessionKafkaSpout.java new file mode 100644 index 0000000..081c3a0 --- /dev/null +++ b/src/main/java/cn/ac/iie/storm/spout/SessionKafkaSpout.java @@ -0,0 +1,71 @@ +package cn.ac.iie.storm.spout; + +import cn.ac.iie.storm.utils.config.StreamAggregateConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.log4j.Logger; +import org.apache.storm.spout.SpoutOutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichSpout; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; + +import java.util.Collections; +import java.util.Map; +import java.util.Properties; + +public class SessionKafkaSpout extends BaseRichSpout { + private static final long serialVersionUID = -3363788553406229592L; + private KafkaConsumer consumer; + private SpoutOutputCollector collector = null; + private TopologyContext context = null; + private final static Logger logger = Logger.getLogger(ConnectionKafkaSpout.class); + + private static Properties createConsumerConfig() { + Properties props = new Properties(); + props.put("bootstrap.servers", StreamAggregateConfig.BOOTSTRAP_SERVERS); + props.put("group.id", StreamAggregateConfig.GROUP_ID); + props.put("session.timeout.ms", "60000"); + props.put("max.poll.records", 3000); + props.put("max.partition.fetch.bytes", 31457280); + props.put("auto.offset.reset", StreamAggregateConfig.AUTO_OFFSET_RESET); + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + logger.info("链接kafka"+StreamAggregateConfig.BOOTSTRAP_SERVERS+"成功"); + return props; + } + + @Override + public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) { + this.collector=spoutOutputCollector; + this.context=topologyContext; + Properties prop = createConsumerConfig(); + this.consumer = new KafkaConsumer<>(prop); + this.consumer.subscribe(Collections.singletonList(StreamAggregateConfig.KAFKA_TOPIC_SESSION)); + } + @Override + public void close() { + consumer.close(); + } + @Override + public void nextTuple() { + try { + ConsumerRecords records = consumer.poll(StreamAggregateConfig.KAFKA_COMSUMER_POLL); + Thread.sleep(StreamAggregateConfig.TOPOLOGY_SPOUT_SLEEP_TIME); + for (ConsumerRecord record : records) { + this.collector.emit(new Values(record.value())); + } + } catch (Exception e) { + logger.error("KafkaSpout发送消息出现异常!", e); + e.printStackTrace(); + } + } + + + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + outputFieldsDeclarer.declare(new Fields("session-source")); + } +} diff --git a/src/main/java/cn/ac/iie/storm/spout/TransactionKafkaSpout.java b/src/main/java/cn/ac/iie/storm/spout/TransactionKafkaSpout.java new file mode 100644 index 0000000..6fbdde0 --- /dev/null +++ b/src/main/java/cn/ac/iie/storm/spout/TransactionKafkaSpout.java @@ -0,0 +1,71 @@ +package cn.ac.iie.storm.spout; + +import cn.ac.iie.storm.utils.config.StreamAggregateConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.log4j.Logger; +import org.apache.storm.spout.SpoutOutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.OutputFieldsDeclarer; +import org.apache.storm.topology.base.BaseRichSpout; +import org.apache.storm.tuple.Fields; +import org.apache.storm.tuple.Values; + +import java.util.Collections; +import java.util.Map; +import java.util.Properties; + +public class TransactionKafkaSpout extends BaseRichSpout { + private static final long serialVersionUID = -3363788553406229592L; + private KafkaConsumer consumer; + private SpoutOutputCollector collector = null; + private TopologyContext context = null; + private final static Logger logger = Logger.getLogger(ConnectionKafkaSpout.class); + + private static Properties createConsumerConfig() { + Properties props = new Properties(); + props.put("bootstrap.servers", StreamAggregateConfig.BOOTSTRAP_SERVERS); + props.put("group.id", StreamAggregateConfig.GROUP_ID); + props.put("session.timeout.ms", "60000"); + props.put("max.poll.records", 3000); + props.put("max.partition.fetch.bytes", 31457280); + props.put("auto.offset.reset", StreamAggregateConfig.AUTO_OFFSET_RESET); + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + logger.info("链接kafka"+StreamAggregateConfig.BOOTSTRAP_SERVERS+"成功"); + return props; + } + + @Override + public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) { + this.collector=spoutOutputCollector; + this.context=topologyContext; + Properties prop = createConsumerConfig(); + this.consumer = new KafkaConsumer<>(prop); + this.consumer.subscribe(Collections.singletonList(StreamAggregateConfig.KAFKA_TOPIC_TRANSACTION)); + } + @Override + public void close() { + consumer.close(); + } + @Override + public void nextTuple() { + try { + ConsumerRecords records = consumer.poll(StreamAggregateConfig.KAFKA_COMSUMER_POLL); + Thread.sleep(StreamAggregateConfig.TOPOLOGY_SPOUT_SLEEP_TIME); + for (ConsumerRecord record : records) { + this.collector.emit(new Values(record.value())); + } + } catch (Exception e) { + logger.error("KafkaSpout发送消息出现异常!", e); + e.printStackTrace(); + } + } + + + @Override + public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { + outputFieldsDeclarer.declare(new Fields("transaction-source")); + } +} diff --git a/src/main/java/cn/ac/iie/storm/utils/general/GetMap.java b/src/main/java/cn/ac/iie/storm/utils/general/GetMap.java new file mode 100644 index 0000000..e4c2494 --- /dev/null +++ b/src/main/java/cn/ac/iie/storm/utils/general/GetMap.java @@ -0,0 +1,195 @@ +package cn.ac.iie.storm.utils.general; + +import com.zdjizhi.utils.StringUtil; +import org.apache.logging.log4j.util.PropertiesUtil; + +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +public class GetMap { + private static Properties propService = new Properties(); + private static Properties propService1 = new Properties(); + + + private static Properties propService_city = new Properties(); + private static Properties propService_province = new Properties(); + private static Properties propService_country = new Properties(); + private static Properties propService_jstx = new Properties(); + + + private static Map labelMap; + private static Map appMap; + private static Map cityMap; + private static Map provinceMap; + private static Map countryMap; + private static Map jstxIdMap; + + + + public static String getJstxDetail_id(int common_app_id){ + String appId = String.valueOf(common_app_id); + if (appMap.get(appId) != null) { + return appMap.get(appId); + } else { + return appId; + } + } + + public static String getProto_id(String common_schema_type){ + return labelMap.get(common_schema_type); + } + + public static String getDy(String location){ + String[] split; + if(location==null){ + return "0"; + } else if (location.contains("..")){ + + //处理新疆省内location逻辑。 + split = location.split("\\.\\."); + if (split.length == 2) { + String xjCity = split[0]; + if (xjCity.length()<1) { + String xjCounty = split[1]; + if (xjCounty == null) { + xjCounty = "0"; + } + return xjCounty; + } + return xjCity; + }else { + return "0"; + } + } else { + //处理新疆省外location逻辑 + //消除location中的空格 + String address = location.replace(" ",""); + String thirdLevelAdd = cityMap.get(address); + if (StringUtil.isNotBlank(thirdLevelAdd)){ + System.out.println("执行的这里"); + return thirdLevelAdd; + }else { + String[] add = address.split("\\."); + if (StringUtil.isNotBlank( provinceMap.get(add[1]))){ + return provinceMap.get(add[1]); + }else if (StringUtil.isNotBlank(countryMap.get(add[0]))){ + return countryMap.get(add[0]); + }else { + return "0"; + } + } + } + } + + + public static String getFjSzd(String location,String mail_from,String mail_protocol_type) { + if (mail_from != null&& mail_protocol_type.equals("SMTP")) { + String[] split; + if (location == null) { + return String.valueOf(3); + } else if (location.contains(",,")) { + return String.valueOf(1); + } else { + split = location.split(","); + if (split.length == 3) { + String country = split[2].replace(" ", ""); + String s = countryMap.get(country); + if (country.equals("China")) { + return String.valueOf(1); + } else if (s != null) { + return String.valueOf(2); + } else { + return String.valueOf(3); + } + } else { + return String.valueOf(3); + } + } + }else { + return String.valueOf(3); + } + } + + + public static String getSjSzd(String location,String mail_to,String mail_protocol_type) { + if (mail_to != null&&(mail_protocol_type.equals("POP3")||mail_protocol_type.equals("IMAP"))) { + String[] split; + if (location == null) { + return String.valueOf(3); + } else if (location.contains(",,")) { + return String.valueOf(1); + } else { + split = location.split(","); + if (split.length == 3) { + String country = split[2].replace(" ", ""); + String s = countryMap.get(country); + if (country.equals("China")) { + return String.valueOf(1); + } else if (s != null) { + return String.valueOf(2); + } else { + return String.valueOf(3); + } + } else { + return String.valueOf(3); + } + } + }else { + return String.valueOf(3); + } + } + + + + public static String getDns_name(String dns_qname){ + if (dns_qname==null){ + return ""; + }else if (dns_qname.contains(" ")){ + return ""; + }else { + return dns_qname; + } + } + + + static { + try { + InputStream applabel = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_app_label_coding.properties"); + InputStream app = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_app_id_coding.properties"); + InputStream jstxid = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_jstx_id_coding.properties"); + + InputStream cityNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_city_coding.properties"); + InputStream provinceNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_province_coding.properties"); + InputStream countryNumbering = PropertiesUtil.class.getClassLoader().getResourceAsStream("common_country_coding.properties"); + + + propService1.load(app); + propService.load(applabel); + propService_jstx.load(jstxid); + + propService_city.load(cityNumbering); + propService_province.load(provinceNumbering); + propService_country.load(countryNumbering); + + + appMap = new HashMap((Map)propService1); + labelMap = new HashMap((Map) propService); + jstxIdMap = new HashMap((Map) propService_jstx); + cityMap = new HashMap((Map) propService_city); + provinceMap = new HashMap((Map) propService_province); + countryMap = new HashMap((Map) propService_country); + } catch (Exception e) { +// propCommon = null; + propService = null; + propService1 = null; + propService_jstx = null; + propService_city = null; + propService_province = null; + propService_country = null; + } + } + + +} diff --git a/src/test/java/TestUFT.java b/src/test/java/TestUFT.java new file mode 100644 index 0000000..b0cba09 --- /dev/null +++ b/src/test/java/TestUFT.java @@ -0,0 +1,10 @@ +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; + +public class TestUFT { + public static void main(String[] args) throws UnsupportedEncodingException { + String str = "�й�"; + System.out.println(new String( str.getBytes("GBK") , StandardCharsets.UTF_8)); + } + +} diff --git a/src/test/java/TestUnicode.java b/src/test/java/TestUnicode.java new file mode 100644 index 0000000..a623898 --- /dev/null +++ b/src/test/java/TestUnicode.java @@ -0,0 +1,15 @@ +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; + +public class TestUnicode { + public static void main(String[] args) throws UnsupportedEncodingException { + String str ="中国"; + String utf8 = new String(str.getBytes(StandardCharsets.UTF_8)); + System.out.println(utf8); + + String unicode = new String (str.getBytes(), StandardCharsets.UTF_8); + System.out.println(unicode); + String gbk = new String(unicode.getBytes("GBK")); + System.out.println(gbk); + } +} diff --git a/xj-log-etl.iml b/xj-log-etl.iml new file mode 100644 index 0000000..7925e44 --- /dev/null +++ b/xj-log-etl.iml @@ -0,0 +1,336 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file