gfwleak/galaxy-tsg-olap-storm-log-stream-completion
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新hbase配置

Browse Source
This commit is contained in:
qidaijie
2019-11-15 09:49:51 +08:00
parent 692eabd056
commit 8f59751da9
16 changed files with 1962 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
properties/core-site.xml Normal file
View File

@@ -0,0 +1,71 @@
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://ns1</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/opt/hadoop/tmp</value>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131702</value>
</property>
<property>
<name>hadoop.proxyuser.root.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.root.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.logfile.size</name>
<value>10000000</value>
<description>The max size of each log file</description>
</property>
<property>
<name>hadoop.logfile.count</name>
<value>1</value>
<description>The max number of log files</description>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>master:2181,slave1:2181,slave2:2181</value>
</property>
<property>  
    <name>fs.hdfs.impl</name>  
    <value>org.apache.hadoop.hdfs.DistributedFileSystem</value>  
    <description>The FileSystem for hdfs: uris.</description>  
</property>
<property>
<name>io.compression.codecs</name>
<value>com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec</value>
</property>
<property>
<name>io.compression.codec.lzo.class</name>
<value>com.hadoop.compression.lzo.LzoCodec</value>
</property>
</configuration>

34
properties/hbase-site.xml
View File

@@ -23,7 +23,7 @@
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://ns1/hbase/hbase-1.4.9</value>
<value>hdfs://ns1/hbase-1.4.9</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
@@ -31,13 +31,13 @@
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>master,slave1,slave2</value>
<value>192.168.40.119,192.168.40.122,192.168.40.123</value>
</property>
<property>
<name>hbase.master.info.port</name>
<value>60010</value>
</property>
#开启schema支持 对应hbase的namespace
<!-- 开启启schema支持 对应hbase的namespace -->
<property>
<name>phoenix.schema.isNamespaceMappingEnabled</name>
<value>true</value>
@@ -46,4 +46,32 @@
<name>phoenix.schema.mapSystemTablesToNamespace</name>
<value>true</value>
</property>
<property>
<name>hbase.client.keyvalue.maxsize</name>
<value>99428800</value>
</property>
<property>
<name>hbase.server.keyvalue.maxsize</name>
<value>99428800</value>
</property>
<property>
<name>hbase.regionserver.wal.codec</name>
<value>org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec</value>
</property>
<property>
<name>phoenix.query.timeoutMs</name>
<value>1800000</value>
</property>
<property>
<name>hbase.rpc.timeout</name>
<value>1200000</value>
</property>
<property>
<name>hbase.client.scanner.caching</name>
<value>1000</value>
</property>
<property>
<name>hbase.client.scanner.timeout.period</name>
<value>1200000</value>
</property>
</configuration>

24
properties/hdfs-site.xml
View File

@@ -17,13 +17,13 @@
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/home/bigdata/hadoop/dfs/name</value>
<value>file:/home/ceiec/hadoop/dfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/home/bigdata/hadoop/dfs/data</value>
<value>file:/home/ceiec/hadoop/dfs/data</value>
</property>
<property>
<name>dfs.replication</name>
@@ -31,11 +31,11 @@
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>master:9001</value>
<value>192.168.40.119:9001</value>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.permissions</name>
@@ -60,32 +60,32 @@
<!-- nn1的RPC通信地址，nn1所在地址 -->
<property>
<name>dfs.namenode.rpc-address.ns1.nn1</name>
<value>master:9000</value>
<value>192.168.40.119:8020</value>
</property>
<!-- nn1的http通信地址，外部访问地址 -->
<property>
<name>dfs.namenode.http-address.ns1.nn1</name>
<value>master:50070</value>
<value>192.168.40.119:50070</value>
</property>
<!-- nn2的RPC通信地址，nn2所在地址 -->
<property>
<name>dfs.namenode.rpc-address.ns1.nn2</name>
<value>slave1:9000</value>
<value>192.168.40.122:8020</value>
</property>
<!-- nn2的http通信地址，外部访问地址 -->
<property>
<name>dfs.namenode.http-address.ns1.nn2</name>
<value>slave1:50070</value>
<value>192.168.40.122:50070</value>
</property>
<!-- 指定NameNode的元数据在JournalNode日志上的存放位置(一般和zookeeper部署在一起) -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://slave1:8485;slave2:8485;master:8485/ns1</value>
<value>qjournal://192.168.40.119:8485;192.168.40.122:8485;192.168.40.123:8485/ns1</value>
</property>
<!-- 指定JournalNode在本地磁盘存放数据的位置 -->
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/home/bigdata/hadoop/journal</value>
<value>/home/ceiec/hadoop/journal</value>
</property>
<!--客户端通过代理访问namenode，访问文件系统，HDFS 客户端与Active 节点通信的Java 类，使用其确定Active 节点是否活跃 -->
<property>

20
properties/service_flow_config.properties
View File

@@ -1,11 +1,11 @@
#管理kafka地址
bootstrap.servers=192.168.40.152:9092
bootstrap.servers=192.168.40.119:9092,192.168.40.122:9092,192.168.40.123:9092
#zookeeper 地址
zookeeper.servers=192.168.40.152:2181
zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
#hbase zookeeper地址
hbase.zookeeper.servers=192.168.40.203:2181,192.168.40.206:2181
hbase.zookeeper.servers=192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181
#hbase tablename
hbase.table.name=subcriber_info
@@ -14,25 +14,25 @@ hbase.table.name=subcriber_info
auto.offset.reset=latest
#kafka broker下的topic名称
kafka.topic=SECURITY-POLICY-LOG
kafka.topic=SECURITY-EVENT-LOG
#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
group.id=security-policy-191112
#输出topic
results.output.topic=SECURITY-POLICY-COMPLETED-LOG
results.output.topic=SECURITY-EVENT-COMPLETED-LOG
#storm topology workers
topology.workers=1
topology.workers=2
#spout并行度 建议与kafka分区数相同
spout.parallelism=1
spout.parallelism=3
#处理补全操作的bolt并行度-worker的倍数
datacenter.bolt.parallelism=1
datacenter.bolt.parallelism=10
#写入kafka的并行度
kafka.bolt.parallelism=1
#写入kafka的并行度10
kafka.bolt.parallelism=10
#定位库地址
ip.library=/home/ceiec/topology/dat/