gfwleak/galaxy-deployment-updata-record
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加hbase hos配置文件

Browse Source
This commit is contained in:
houjinchuan
2023-11-10 15:41:49 +08:00
parent 84d6fa03f6
commit 4841bcdff6
4 changed files with 800 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

271
TSG发布版本更新记录/TSG-23.10/hbase/hbase-site.xml Normal file
View File

@@ -0,0 +1,271 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://ns1/hbase/hbase-2.2.3</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>192.168.44.11,192.168.44.14,192.168.44.15</value>
</property>
<property>
<name>hbase.zookeeper.property.clientPort</name>
<value>2181</value>
</property>
<property>
<name>hbase.master.info.port</name>
<value>60010</value>
</property>
<property>
<name>hbase.server.keyvalue.maxsize</name>
<value>5368709120</value>
</property>
<property>
<name>zookeeper.znode.parent</name>
<value>/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.rpc.timeout</name>
<value>300000</value>
<description>This is for the RPC layer to define how long (millisecond) HBase client applications
take for a remote call to time out. It uses pings to check connections
but will eventually throw a TimeoutException.</description>
</property>
<property>
<name>zookeeper.session.timeout</name>
<value>300000</value>
</property>
<property>
<name>hbase.regionserver.maxlogs</name>
<value>64</value>
</property>
<!--超过该值的文件将在mob compaction中合并-->
<property>
<name>hbase.mob.compaction.mergeable.threshold</name>
<value>1342177280</value>
</property>
<property>
<name>hbase.mob.file.cache.size</name>
<value>10000</value>
<description>
Number of opened file handlers to cache.
A larger value will benefit reads by providing more file handlers per mob
file cache and would reduce frequent file opening and closing.
However, if this is set too high, this could lead to a "too many opened file handers"
The default value is 1000.
</description>
</property>
<!--mob cache回收缓存周期-->
<property>
<name>hbase.mob.cache.evict.period</name>
<value>3600</value>
<description>
The amount of time in seconds after which an unused file is evicted from the
MOB cache. The default value is 3600 seconds.
</description>
</property>
<!--mob cache回收之后cache中保留文件个数比例，cache数量超过hbase.mob.file.cache.size会回收-->
<property>
<name>hbase.mob.cache.evict.remain.ratio</name>
<value>0.5f</value>
<description>
A multiplier (between 0.0 and 1.0), which determines how many files remain cached
after the threshold of files that remains cached after a cache eviction occurs
which is triggered by reaching the `hbase.mob.file.cache.size` threshold.
The default value is 0.5f, which means that half the files (the least-recently-used
ones) are evicted.
</description>
</property>
<!--开启mob-->
<property>
<name>hfile.format.version</name>
<value>3</value>
</property>
<property>
<name>hbase.hregion.memstore.flush.size</name>
<value>33554432</value>
<description>
memstore的大小超过该限制（单位：byte）后将被flush到磁盘。这个大小由一个线程间断性的检查，检查的间隔由
hbase.server.thread.wakefrequency决定
</description>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>2</value>
</property>
<property>
<name>hbase.regionserver.flush.check.period</name>
<value>10000</value>
<description>
It determines the flushing check period of PeriodicFlusher in regionserver.
If unset, it uses hbase.server.thread.wakefrequency as default value.
(in milliseconds)
</description>
</property>
<property>
<name>hbase.regionserver.compaction.check.period</name>
<value>10000</value>
</property>
<property>
<name>hbase.regionserver.global.memstore.size.lower.limit</name>
<value>0.95</value>
</property>
<property>
<name>hbase.regionserver.global.memstore.size</name>
<value>0.45</value>
</property>
<property>
<name>hfile.block.cache.size</name>
<value>0.3</value>
<description>
Percentage of maximum heap (-Xmx setting) to allocate to block cache
used by a StoreFile. Default of 0.4 means allocate 40%.
Set to 0 to disable but it's not recommended; you need at least
enough cache to hold the storefile indices.
</description>
</property>
<property>
<name>hbase.hregion.memstore.block.multiplier</name>
<value>2</value>
</property>
<property>
<name>hbase.ipc.server.max.callqueue.length</name>
<value>1073741824</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>10</value>
<description>
Count of RPC Listener instances spun up on RegionServers.
Same property is used by the Master for count of master handlers.
</description>
</property>
<property>
<name>hbase.zookeeper.property.maxClientCnxns</name>
<value>1000</value>
</property>
<property>
<name>hbase.ipc.max.request.size</name>
<value>1173741824</value>
</property>
<property>
<name>hbase.hstore.blockingWaitTime</name>
<value>30000</value>
<description>
The time an HRegion will block updates for after hitting the StoreFile
limit defined by hbase.hstore.blockingStoreFiles.
After this time has elapsed, the HRegion will stop blocking updates even
if a compaction has not been completed.
</description>
</property>
<property>
<name>hbase.hstore.blockingStoreFiles</name>
<value>100</value>
<description>
If more than this number of StoreFiles in any one Store
(one StoreFile is written per flush of MemStore) then updates are
blocked for this HRegion until a compaction is completed, or
until hbase.hstore.blockingWaitTime has been exceeded.
</description>
</property>
<property>
　 <name>hbase.hregion.max.filesize</name>
　 <value>3221225472</value>
</property>
<property>
<name>hbase.regionserver.regionSplitLimit</name>
<value>1000</value>
</property>
<property>
<name>hbase.replication</name>
<value>true</value>
</property>
<property>
<name>phoenix.schema.isNamespaceMappingEnabled</name>
<value>true</value>
</property>
<property>
<name>phoenix.schema.mapSystemTablesToNamespace</name>
<value>true</value>
</property>
<property>
<name>hbase.coprocessor.master.classes</name>
<value>org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint</value>
</property>
<property>
<name>hbase.master.loadbalancer.class</name>
<value>org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer</value>
</property>
<!--表region自动平衡-->
<property>
  <name>hbase.master.loadbalance.bytable</name>
  <value>true</value>
</property>
</configuration>

120
TSG发布版本更新记录/TSG-23.10/hos/galaxy-hos-service.yml Normal file
View File

@@ -0,0 +1,120 @@
server:
port: 8186
tomcat:
max-threads: 400
#tomcat缓存大小，单位KB系统默认10M，配置10g
tomcat:
cacheMaxSize: 1000000
#hbase参数
hbase:
zookeeper_quorum: 192.168.44.11,192.168.44.14,192.168.44.15
zookeeper_property_clientPort: 2181
zookeeper_znode_parent: /hbase
client_retries_number: 9
rpc_timeout: 100000
connect_pool: 10
client_write_buffer: 10485760
client_keyvalue_maxsize: 1024000000
#批量获取数量
get_batch: 10000
#part 最大数据量
maxParts: 100000
#每次获取的part数
get_part_batch: 1000
#每次追加是否更新主文件
isUpdate: 1
#hbase索引表前缀，前缀为以下的都为索引表
time_index_table_prefix: index_time_
filename_index_table_prefix: index_filename_
partfile_index_table_prefix: index_partfile_
system_bucket_meta: system:bucket_meta
#创建表预分区时的分区，为空则不分区
region_start_key: 1,2,3,4,5,6,7,8,9,a,b,c,d,e,f
filename_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f
part_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f
#获取文件大小的目录
hbasePath: /hbase/hbase-2.2.3
#1是集群0是单机，主要针对存储配额获取方式
standone: 1
#hadoop集群namenode节点
namenodes: 192.168.44.11,192.168.44.14
#hadoop端口
hadoop_port: 9000
hadoop_user: root
hadoop_defaultFS: hdfs://ns1
hadoop_nameservices: ns1
hadoop_namenodes_ns1: nn1,nn2
hadoop_replication: 2
#建表时是否打开hbase wal，1打开，0关闭
openWal: 0
#ttl相关参数
ttl_scan_batch: 500
ttl_scan_caching: 1000
ttl_delete_batch: 1000
#是否打开验证，0打开，打开需要使用S3身份验证或者token访问服务
auth:
open: 0
#http访问使用的token
token: ENC(vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo)
#s3验证
s3:
accessKey: ENC(FUQDvVP+zqCiwHQhXcRvbw==)
secretKey: ENC(FUQDvVP+zqCiwHQhXcRvbw==)
hos:
#长连接超时时间
keepAliveTimeout: 60000
#批量删除对象的最大数量
deleteMultipleNumber: 1000
#获取对象列表等操作的最大值
maxResultLimit: 100000
#分块上传的最大分块数
maxPartNumber: 10000
#追加上传的最大次数
maxPosition: 100000
#存放对象的用户自定义元数据的请求头
metaHeader: x-hos-meta-message
#存放对象信息的请求头
objectInfo: x-hos-object-info
#是否快速下载文件，1打开，hbase内存小于20G的集群设为0
isQuickDownloadFile: 0
#是否打开对象列表查询功能，1打开
simple: 1
#用户白名单（hbase的namespace），获取存储配额
users: default
#元数据存储占比
metaProportion: 0.03
#是否打开限流,0:关闭，1:打开
openRateLimiter: 1
#限流每秒请求数
rateLimiterQps: 20000
#展示追加文件丢失块的最大数量
lostPartsCount: 10
#执行ttl的线程数
thread: 10
#是否打开手动ttl，1打开，默认为1
manualTtl: 0
#文件最大值
maxFileSize: 5368709000
#小文件阈值
uploadThreshold: 10485760
#设置上传文件大小的最大值
spring:
servlet:
multipart:
max-file-size: 1024MB
max-request-size: 1024MB
#Prometheus参数
application:
name: HosServiceApplication
#Prometheus参数
management:
endpoints:
web:
exposure:
include: '*'
metrics:
tags:
application: ${spring.application.name}
logging:
config: ./config/log4j2-dev.xml

302
TSG发布版本更新记录/TSG-23.11/hbase/hbase-site.xml Normal file
View File

@@ -0,0 +1,302 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://ns1/hbase/hbase-2.2.3</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>192.168.44.11,192.168.44.14,192.168.44.15</value>
</property>
<property>
<name>hbase.zookeeper.property.clientPort</name>
<value>2181</value>
</property>
<property>
<name>hbase.master.info.port</name>
<value>60010</value>
</property>
<property>
<name>hbase.server.keyvalue.maxsize</name>
<value>1073741824</value>
</property>
<property>
<name>zookeeper.znode.parent</name>
<value>/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.rpc.timeout</name>
<value>300000</value>
<description>This is for the RPC layer to define how long (millisecond) HBase client applications
take for a remote call to time out. It uses pings to check connections
but will eventually throw a TimeoutException.</description>
</property>
<property>
<name>zookeeper.session.timeout</name>
<value>300000</value>
</property>
<property>
<name>hbase.regionserver.maxlogs</name>
<value>64</value>
</property>
<!--超过该值的文件将在mob compaction中合并-->
<property>
<name>hbase.mob.compaction.mergeable.threshold</name>
<value>1342177280</value>
</property>
<property>
<name>hbase.mob.file.cache.size</name>
<value>10000</value>
<description>
Number of opened file handlers to cache.
A larger value will benefit reads by providing more file handlers per mob
file cache and would reduce frequent file opening and closing.
However, if this is set too high, this could lead to a "too many opened file handers"
The default value is 1000.
</description>
</property>
<!--mob cache回收缓存周期-->
<property>
<name>hbase.mob.cache.evict.period</name>
<value>3600</value>
<description>
The amount of time in seconds after which an unused file is evicted from the
MOB cache. The default value is 3600 seconds.
</description>
</property>
<!--mob cache回收之后cache中保留文件个数比例，cache数量超过hbase.mob.file.cache.size会回收-->
<property>
<name>hbase.mob.cache.evict.remain.ratio</name>
<value>0.5f</value>
<description>
A multiplier (between 0.0 and 1.0), which determines how many files remain cached
after the threshold of files that remains cached after a cache eviction occurs
which is triggered by reaching the `hbase.mob.file.cache.size` threshold.
The default value is 0.5f, which means that half the files (the least-recently-used
ones) are evicted.
</description>
</property>
<!--开启mob-->
<property>
<name>hfile.format.version</name>
<value>3</value>
</property>
<property>
<name>hbase.hregion.memstore.flush.size</name>
<value>134217728</value>
</property>
<property>
<name>hbase.hstore.flusher.count</name>
<value>4</value>
</property>
<property>
<name>hbase.regionserver.flush.check.period</name>
<value>10000</value>
<description>
It determines the flushing check period of PeriodicFlusher in regionserver.
If unset, it uses hbase.server.thread.wakefrequency as default value.
(in milliseconds)
</description>
</property>
<property>
<name>hbase.regionserver.compaction.check.period</name>
<value>10000</value>
</property>
<property>
<name>hbase.regionserver.global.memstore.size.lower.limit</name>
<value>0.95</value>
</property>
<property>
<name>hbase.regionserver.global.memstore.size</name>
<value>0.45</value>
</property>
<property>
<name>hfile.block.cache.size</name>
<value>0.3</value>
</property>
<property>
<name>hbase.hregion.memstore.block.multiplier</name>
<value>2</value>
</property>
<property>
<name>hbase.ipc.server.max.callqueue.length</name>
<value>1073741824</value>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>10</value>
<description>
Count of RPC Listener instances spun up on RegionServers.
Same property is used by the Master for count of master handlers.
</description>
</property>
<property>
<name>hbase.zookeeper.property.maxClientCnxns</name>
<value>1000</value>
</property>
<property>
<name>hbase.ipc.max.request.size</name>
<value>1173741824</value>
</property>
<property>
<name>hbase.hstore.blockingWaitTime</name>
<value>20000</value>
<description>
The time an HRegion will block updates for after hitting the StoreFile
limit defined by hbase.hstore.blockingStoreFiles.
After this time has elapsed, the HRegion will stop blocking updates even
if a compaction has not been completed.
</description>
</property>
<property>
<name>hbase.hstore.blockingStoreFiles</name>
<value>100</value>
<description>
If more than this number of StoreFiles in any one Store
(one StoreFile is written per flush of MemStore) then updates are
blocked for this HRegion until a compaction is completed, or
until hbase.hstore.blockingWaitTime has been exceeded.
</description>
</property>
<property>
　 <name>hbase.hregion.max.filesize</name>
　 <value>10737418240‬</value>
</property>
<property>
<name>hbase.regionserver.regionSplitLimit</name>
<value>500</value>
</property>
<property>
<name>phoenix.schema.isNamespaceMappingEnabled</name>
<value>true</value>
</property>
<property>
<name>phoenix.schema.mapSystemTablesToNamespace</name>
<value>true</value>
</property>
<property>
<name>hbase.coprocessor.master.classes</name>
<value>org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint</value>
</property>
<property>
<name>hbase.master.loadbalancer.class</name>
<value>org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer</value>
</property>
<!--表region自动平衡-->
<property>
  <name>hbase.master.loadbalance.bytable</name>
  <value>true</value>
</property>
<!-- storefile数量大于该值执行compact -->
<property>
<name>hbase.hstore.compactionThreshold</name>
<value>5</value>
</property>
<property>
<name>hbase.hstore.compaction.min</name>
<value>5</value>
</property>
<!-- 最多选取多少个storefile进行compace -->
<property>
<name>hbase.hstore.compaction.max</name>
<value>20</value>
</property>
<property>
<name>hbase.hstore.compaction.min.size</name>
<value>134217728</value>
</property>
<property>
<name>hbase.hstore.compaction.max.size</name>
<value>10737418240</value>
</property>
<property>
<name>hbase.regionserver.thread.compaction.throttle</name>
<value>2684354560</value>
</property>
<property>
<name>hbase.regionserver.thread.compaction.small</name>
<value>5</value>
</property>
<property>
<name>hbase.regionserver.thread.compaction.large</name>
<value>5</value>
</property>
<property>
<name>hbase.hstore.compaction.kv.max</name>
<value>10</value>
</property>
<property>
<name>hbase.mob.compaction.threads.max</name>
<value>2</value>
</property>
<property>
<name>hbase.hregion.majorcompaction</name>
<value>604800000</value>
</property>
</configuration>

107
TSG发布版本更新记录/TSG-23.11/hos/galaxy-hos-service.yml Normal file
View File

@@ -0,0 +1,107 @@
#服务端口
server:
port: 8186
tomcat:
max-threads: 400
#tomcat缓存大小，单位KB系统默认10M，配置10g
tomcat:
cacheMaxSize: 1000000
#hbase参数
hbase:
zookeeper_quorum: 192.168.44.11,192.168.44.14,192.168.44.15
zookeeper_port: 2181
zookeeper_znode_parent: /hbase
client_retries_number: 9
rpc_timeout: 100000
connect_pool: 10
client_write_buffer: 10485760
client_key_value_maxsize: 1073741824
mob_threshold: 10485760
#part 最大数据量
max_parts: 100000
#每次获取的part数
get_part_batch: 10
#hbase索引表前缀，前缀为以下的都为索引表
time_index_table_prefix: index_time_
filename_index_table_prefix: index_filename_
partfile_index_table_prefix: index_partfile_
system_bucket_meta: system:bucket_meta
#创建表预分区时的分区，为空则不分区
region_start_key: 1,2,3,4,5,6,7,8,9,a,b,c,d,e,f
filename_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f
part_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f
#获取文件大小的目录
data_path: /hbase
#1是集群0是单机
standalone: 1
#hadoop集群namenode节点
hadoop_name_nodes: 192.168.44.11,192.168.44.14
#hadoop端口
hadoop_port: 9000
hadoop_user: root
hadoop_default_fs: hdfs://ns1
hadoop_name_services: ns1
hadoop_name_nodes_ns1: nn1,nn2
hadoop_replication: 2
#ttl相关参数
ttl_scan_batch: 1000
ttl_scan_caching: 1000
ttl_delete_batch: 1000
#是否打开验证，0打开，打开需要使用S3身份验证或者token访问服务
auth:
open: 0
#http访问使用的token
token: ENC(vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo)
#s3验证
s3:
accesskey: ENC(FUQDvVP+zqCiwHQhXcRvbw==)
secretkey: ENC(FUQDvVP+zqCiwHQhXcRvbw==)
hos:
#文件大小阈值
maxFileSize: 5073741800
#大文件阈值
uploadThreshold: 104857600
#长连接超时时间
keepAliveTimeout: 60000
#批量删除对象的最大数量
deleteMultipleNumber: 1000
#获取对象列表等操作的最大值
maxResultLimit: 100000
#分块上传的最大分块数
maxPartNumber: 10000
#追加上传的最大次数
maxAppendNumber: 100000
#是否快速下载文件，1打开，hbase内存小于20G的集群设为0
isQuickDownloadFile: 0
#用户白名单（hbase的namespace），获取存储配额
users: default
#元数据存储占比
metaProportion: 0.02
#是否打开限流,0:关闭，1:打开
openRateLimiter: 0
#限流每秒请求数
rateLimiterQps: 20000
#是否打开手动ttl，1打开，默认为1
manualTtl: 1
#执行ttl的线程数
ttlThread: 10
#设置上传文件大小的最大值
spring:
servlet:
multipart:
max-file-size: 1024MB
max-request-size: 1024MB
#Prometheus参数
application:
name: HosServiceApplication
#Prometheus参数
management:
endpoints:
web:
exposure:
include: '*'
metrics:
tags:
application: ${spring.application.name}
logging:
config: ./config/log4j2-dev.xml