k8s初始版本storm

storm/topo/storm_topology/completion/alone/CONNECTION-RECORD-LOG

@@ -0,0 +1,75 @@
+#管理kafka地址
+#bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092
+bootstrap.servers=ipaddress:9092
+#zookeeper 地址
+#zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
+zookeeper.servers=ipaddress:2181
+#hbase zookeeper地址
+#hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
+hbase.zookeeper.servers=ipaddress:2182
+#hbase tablename
+hbase.table.name=subcriber_info
+
+#latest/earliest
+auto.offset.reset=latest
+
+#kafka broker下的topic名称
+kafka.topic=CONNECTION-RECORD-LOG
+
+#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
+group.id=connection-log-191122
+
+#输出topic
+results.output.topic=CONNECTION-RECORD-COMPLETED-LOG
+
+#storm topology workers
+topology.workers=1
+
+#spout并行度 建议与kafka分区数相同
+spout.parallelism=1
+
+#处理补全操作的bolt并行度-worker的倍数
+datacenter.bolt.parallelism=1
+
+#写入kafka的并行度
+kafka.bolt.parallelism=1
+
+#定位库地址
+ip.library=/dat/
+
+#kafka批量条数
+batch.insert.num=2000
+
+#数据中心（UID）
+data.center.id.num=12
+
+#tick时钟频率
+topology.tick.tuple.freq.secs=5
+
+#hbase 更新时间
+hbase.tick.tuple.freq.secs=60
+
+#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效
+topology.config.max.spout.pending=150000
+
+#ack设置 1启动ack 0不启动ack
+topology.num.acks=0
+
+#spout接收睡眠时间
+topology.spout.sleep.time=1
+
+#用于过滤对准用户名
+check.ip.scope=10,100,192
+
+#允许发送kafka最大失败数
+max.failure.num=20
+
+#influx地址
+influx.ip=http://192.168.40.151:8086
+
+#influx用户名
+influx.username=admin
+
+#influx密码
+influx.password=admin
+

storm/topo/storm_topology/completion/alone/start.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+JAR_NAME='log-stream-completion.jar'
+storm jar $JAR_NAME cn.ac.iie.topology.LogFlowWriteTopology $1 remote

storm/topo/storm_topology/completion/alone/startall.sh

@@ -0,0 +1,34 @@
+#! /bin/bash
+#启动storm任务脚本
+
+#任务jar所在目录
+BASE_DIR=$2
+#jar name
+JAR_NAME='log-stream-completion.jar'
+#nimbus ip
+LOCAL_IP=$3
+
+cd $BASE_DIR
+
+function read_dir(){
+        for file in `ls $1` #注意此处这是两个反引号，表示运行系统命令
+ do
+        if [ -d $1"/"$file ] #注意此处之间一定要加上空格，否则会报错
+                then
+                        read_dir $1"/"$file
+                else
+                        jar -xvf $BASE_DIR/$JAR_NAME service_flow_config.properties
+			cat $1$file > $BASE_DIR/service_flow_config.properties
+                        sed -i 's/ipaddress/'$LOCAL_IP'/' $BASE_DIR/service_flow_config.properties
+			jar -uvf $BASE_DIR/$JAR_NAME service_flow_config.properties
+                        kubectl exec -c k8s-nimbus nimbus-0 -n default -- storm jar /opt/test/topo/storm_topology/completion/alone/$JAR_NAME cn.ac.iie.topology.LogFlowWriteTopology TEST1 remote
+	fi
+done
+}
+if [ $# != 3 ];then
+    echo "usage: ./startall.sh [Configuration path] [Path of jar] [nimbus ip]"
+    exit 1
+fi
+#读取第一个参数 为配置文件目录名称
+read_dir $1
+

storm/topo/storm_topology/completion/completionconfig/CONNECTION-RECORD-LOG

@@ -0,0 +1,75 @@
+#管理kafka地址
+#bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092
+bootstrap.servers=ipaddress:9092
+#zookeeper 地址
+#zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
+zookeeper.servers=ipaddress:2181
+#hbase zookeeper地址
+#hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
+hbase.zookeeper.servers=ipaddress:2182
+#hbase tablename
+hbase.table.name=subcriber_info
+
+#latest/earliest
+auto.offset.reset=latest
+
+#kafka broker下的topic名称
+kafka.topic=CONNECTION-RECORD-LOG
+
+#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
+group.id=connection-log-191122
+
+#输出topic
+results.output.topic=CONNECTION-RECORD-COMPLETED-LOG
+
+#storm topology workers
+topology.workers=1
+
+#spout并行度 建议与kafka分区数相同
+spout.parallelism=1
+
+#处理补全操作的bolt并行度-worker的倍数
+datacenter.bolt.parallelism=1
+
+#写入kafka的并行度
+kafka.bolt.parallelism=1
+
+#定位库地址
+ip.library=/dat/
+
+#kafka批量条数
+batch.insert.num=2000
+
+#数据中心（UID）
+data.center.id.num=12
+
+#tick时钟频率
+topology.tick.tuple.freq.secs=5
+
+#hbase 更新时间
+hbase.tick.tuple.freq.secs=60
+
+#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效
+topology.config.max.spout.pending=150000
+
+#ack设置 1启动ack 0不启动ack
+topology.num.acks=0
+
+#spout接收睡眠时间
+topology.spout.sleep.time=1
+
+#用于过滤对准用户名
+check.ip.scope=10,100,192
+
+#允许发送kafka最大失败数
+max.failure.num=20
+
+#influx地址
+influx.ip=http://192.168.40.151:8086
+
+#influx用户名
+influx.username=admin
+
+#influx密码
+influx.password=admin
+

storm/topo/storm_topology/completion/completionconfig/PROXY-POLICY-LOG

@@ -0,0 +1,75 @@
+#管理kafka地址
+##bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092
+bootstrap.servers=ipaddress:9092
+##zookeeper 地址
+##zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
+zookeeper.servers=ipaddress:2181
+##hbase zookeeper地址
+##hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
+hbase.zookeeper.servers=ipaddress:2182
+#hbase tablename
+hbase.table.name=subcriber_info
+
+#latest/earliest
+auto.offset.reset=latest
+
+#kafka broker下的topic名称
+kafka.topic=PROXY-EVENT-LOG
+
+#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
+group.id=proxy-event-191122
+
+#输出topic
+results.output.topic=PROXY-EVENT-COMPLETED-LOG
+
+#storm topology workers
+topology.workers=1
+
+#spout并行度 建议与kafka分区数相同
+spout.parallelism=1
+
+#处理补全操作的bolt并行度-worker的倍数
+datacenter.bolt.parallelism=1
+
+#写入kafka的并行度
+kafka.bolt.parallelism=1
+
+#定位库地址
+ip.library=/dat/
+
+#kafka批量条数
+batch.insert.num=2000
+
+#数据中心（UID）
+data.center.id.num=14
+
+#tick时钟频率
+topology.tick.tuple.freq.secs=5
+
+#hbase 更新时间
+hbase.tick.tuple.freq.secs=60
+
+#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效
+topology.config.max.spout.pending=150000
+
+#ack设置 1启动ack 0不启动ack
+topology.num.acks=0
+
+#spout接收睡眠时间
+topology.spout.sleep.time=1
+
+#用于过滤对准用户名
+check.ip.scope=10,100,192
+
+#允许发送kafka最大失败数
+max.failure.num=20
+
+#influx地址
+influx.ip=http://192.168.40.151:8086
+
+#influx用户名
+influx.username=admin
+
+#influx密码
+influx.password=admin
+

storm/topo/storm_topology/completion/completionconfig/RADIUS-RECORD-LOG

@@ -0,0 +1,75 @@
+#管理kafka地址
+##bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092
+bootstrap.servers=ipaddress:9092
+##zookeeper 地址
+##zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
+zookeeper.servers=ipaddress:2181
+##hbase zookeeper地址
+##hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
+hbase.zookeeper.servers=ipaddress:2182
+#hbase tablename
+hbase.table.name=subcriber_info
+
+#latest/earliest
+auto.offset.reset=latest
+
+#kafka broker下的topic名称
+kafka.topic=RADIUS-RECORD-LOG
+
+#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
+group.id=radius-record-191122
+
+#输出topic
+results.output.topic=RADIUS-RECORD-COMPLETED-LOG
+
+#storm topology workers
+topology.workers=1
+
+#spout并行度 建议与kafka分区数相同
+spout.parallelism=1
+
+#处理补全操作的bolt并行度-worker的倍数
+datacenter.bolt.parallelism=1
+
+#写入kafka的并行度
+kafka.bolt.parallelism=1
+
+#定位库地址
+ip.library=/dat/
+
+#kafka批量条数
+batch.insert.num=2000
+
+#数据中心（UID）
+data.center.id.num=13
+
+#tick时钟频率
+topology.tick.tuple.freq.secs=5
+
+#hbase 更新时间
+hbase.tick.tuple.freq.secs=60
+
+#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效
+topology.config.max.spout.pending=150000
+
+#ack设置 1启动ack 0不启动ack
+topology.num.acks=0
+
+#spout接收睡眠时间
+topology.spout.sleep.time=1
+
+#用于过滤对准用户名
+check.ip.scope=10,100,192
+
+#允许发送kafka最大失败数
+max.failure.num=20
+
+#influx地址
+influx.ip=http://192.168.40.151:8086
+
+#influx用户名
+influx.username=admin
+
+#influx密码
+influx.password=admin
+

storm/topo/storm_topology/completion/completionconfig/SECURITY-POLICY-LOG

@@ -0,0 +1,75 @@
+#管理kafka地址
+##bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092
+bootstrap.servers=ipaddress:9092
+##zookeeper 地址
+##zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
+zookeeper.servers=ipaddress:2181
+##hbase zookeeper地址
+##hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
+hbase.zookeeper.servers=ipaddress:2182
+#hbase tablename
+hbase.table.name=subcriber_info
+
+#latest/earliest
+auto.offset.reset=latest
+
+#kafka broker下的topic名称
+kafka.topic=SECURITY-EVENT-LOG
+
+#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
+group.id=security-policy-191122
+
+#输出topic
+results.output.topic=SECURITY-EVENT-COMPLETED-LOG
+
+#storm topology workers
+topology.workers=1
+
+#spout并行度 建议与kafka分区数相同
+spout.parallelism=1
+
+#处理补全操作的bolt并行度-worker的倍数
+datacenter.bolt.parallelism=1
+
+#写入kafka的并行度
+kafka.bolt.parallelism=1
+
+#定位库地址
+ip.library=/dat/
+
+#kafka批量条数
+batch.insert.num=2000
+
+#数据中心（UID）
+data.center.id.num=15
+
+#tick时钟频率
+topology.tick.tuple.freq.secs=5
+
+#hbase 更新时间
+hbase.tick.tuple.freq.secs=60
+
+#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效
+topology.config.max.spout.pending=150000
+
+#ack设置 1启动ack 0不启动ack
+topology.num.acks=0
+
+#spout接收睡眠时间
+topology.spout.sleep.time=1
+
+#用于过滤对准用户名
+check.ip.scope=10,100,192
+
+#允许发送kafka最大失败数
+max.failure.num=20
+
+#influx地址
+influx.ip=http://192.168.40.151:8086
+
+#influx用户名
+influx.username=admin
+
+#influx密码
+influx.password=admin
+

storm/topo/storm_topology/completion/startall.sh

@@ -0,0 +1,34 @@
+#! /bin/bash
+#启动storm任务脚本
+
+#任务jar所在目录
+BASE_DIR=$2
+#jar name
+JAR_NAME='log-stream-completion.jar'
+#nimbus ip
+LOCAL_IP=$3
+
+cd $BASE_DIR
+
+function read_dir(){
+        for file in `ls $1` #注意此处这是两个反引号，表示运行系统命令
+ do
+        if [ -d $1"/"$file ] #注意此处之间一定要加上空格，否则会报错
+                then
+                        read_dir $1"/"$file
+                else
+                        jar -xvf $BASE_DIR/$JAR_NAME service_flow_config.properties
+			cat $1$file > $BASE_DIR/service_flow_config.properties
+                        sed -i 's/ipaddress/'$LOCAL_IP'/' $BASE_DIR/service_flow_config.properties
+			jar -uvf $BASE_DIR/$JAR_NAME service_flow_config.properties
+                        docker run -it --rm -v $BASE_DIR/$JAR_NAME:/$JAR_NAME --env ZK_IPARR=$LOCAL_IP --env NIMBUS_IP=$LOCAL_IP --env ZK_PORTS=2182 --net host storm:1.0.2 /opt/apache-storm-1.0.2/start_storm.sh storm jar /$JAR_NAME cn.ac.iie.topology.LogFlowWriteTopology $file remote
+	fi
+done
+}
+if [ $# != 3 ];then
+    echo "usage: ./startall.sh [Configuration path] [Path of jar] [nimbus ip]"
+    exit 1
+fi
+#读取第一个参数 为配置文件目录名称
+read_dir $1
+

storm/topo/storm_topology/completion/stoptopologe.sh

@@ -0,0 +1,17 @@
+#! /bin/bash
+#storm任务停止脚本
+function read_dir(){
+        for file in `ls $1` #注意此处这是两个反引号，表示运行系统命令
+ do
+        if [ -d $1"/"$file ] #注意此处之间一定要加上空格，否则会报错
+                then
+                    read_dir $1"/"$file
+                else
+	        docker exec nimbus storm kill $file -w 1
+#		/home/bigdata/apache-storm-1.0.2/bin/storm kill $file -w 1 
+               echo $file #在此处处理文件即可
+        fi
+done
+}
+#读取第一个参数 为配置文件目录名
+read_dir $1