diff --git a/TSG发布版本更新记录/TSG-23.10/hbase/hbase-site.xml b/TSG发布版本更新记录/TSG-23.10/hbase/hbase-site.xml new file mode 100644 index 0000000..cc1dacc --- /dev/null +++ b/TSG发布版本更新记录/TSG-23.10/hbase/hbase-site.xml @@ -0,0 +1,271 @@ + + + + + + hbase.rootdir + hdfs://ns1/hbase/hbase-2.2.3 + + + + hbase.cluster.distributed + true + + + + hbase.zookeeper.quorum + 192.168.44.11,192.168.44.14,192.168.44.15 + + + + hbase.zookeeper.property.clientPort + 2181 + + + + hbase.master.info.port + 60010 + + + + hbase.server.keyvalue.maxsize + 5368709120 + + + + zookeeper.znode.parent + /hbase + + + + hbase.cluster.distributed + true + + + + hbase.rpc.timeout + 300000 + This is for the RPC layer to define how long (millisecond) HBase client applications + take for a remote call to time out. It uses pings to check connections + but will eventually throw a TimeoutException. + + + + zookeeper.session.timeout + 300000 + + + + hbase.regionserver.maxlogs + 64 + + + + + hbase.mob.compaction.mergeable.threshold + 1342177280 + + + + hbase.mob.file.cache.size + 10000 + + Number of opened file handlers to cache. + A larger value will benefit reads by providing more file handlers per mob + file cache and would reduce frequent file opening and closing. + However, if this is set too high, this could lead to a "too many opened file handers" + The default value is 1000. + + + + + + hbase.mob.cache.evict.period + 3600 + + The amount of time in seconds after which an unused file is evicted from the + MOB cache. The default value is 3600 seconds. + + + + + + hbase.mob.cache.evict.remain.ratio + 0.5f + + A multiplier (between 0.0 and 1.0), which determines how many files remain cached + after the threshold of files that remains cached after a cache eviction occurs + which is triggered by reaching the `hbase.mob.file.cache.size` threshold. + The default value is 0.5f, which means that half the files (the least-recently-used + ones) are evicted. + + + + + + hfile.format.version + 3 + + + + hbase.hregion.memstore.flush.size + 33554432 + + memstore的大小超过该限制(单位:byte)后将被flush到磁盘。这个大小由一个线程间断性的检查,检查的间隔由 + hbase.server.thread.wakefrequency决定 + + + + + hbase.hstore.flusher.count + 2 + + + + hbase.regionserver.flush.check.period + 10000 + + It determines the flushing check period of PeriodicFlusher in regionserver. + If unset, it uses hbase.server.thread.wakefrequency as default value. + (in milliseconds) + + + + + hbase.regionserver.compaction.check.period + 10000 + + + + hbase.regionserver.global.memstore.size.lower.limit + 0.95 + + + + hbase.regionserver.global.memstore.size + 0.45 + + + + hfile.block.cache.size + 0.3 + + Percentage of maximum heap (-Xmx setting) to allocate to block cache + used by a StoreFile. Default of 0.4 means allocate 40%. + Set to 0 to disable but it's not recommended; you need at least + enough cache to hold the storefile indices. + + + + + hbase.hregion.memstore.block.multiplier + 2 + + + + hbase.ipc.server.max.callqueue.length + 1073741824 + + + + hbase.regionserver.handler.count + 10 + + Count of RPC Listener instances spun up on RegionServers. + Same property is used by the Master for count of master handlers. + + + + + hbase.zookeeper.property.maxClientCnxns + 1000 + + + + hbase.ipc.max.request.size + 1173741824 + + + + hbase.hstore.blockingWaitTime + 30000 + + The time an HRegion will block updates for after hitting the StoreFile + limit defined by hbase.hstore.blockingStoreFiles. + After this time has elapsed, the HRegion will stop blocking updates even + if a compaction has not been completed. + + + + + hbase.hstore.blockingStoreFiles + 100 + + If more than this number of StoreFiles in any one Store + (one StoreFile is written per flush of MemStore) then updates are + blocked for this HRegion until a compaction is completed, or + until hbase.hstore.blockingWaitTime has been exceeded. + + + + +  hbase.hregion.max.filesize +  3221225472 + + + + hbase.regionserver.regionSplitLimit + 1000 + + + + hbase.replication + true + + + + phoenix.schema.isNamespaceMappingEnabled + true + + + + phoenix.schema.mapSystemTablesToNamespace + true + + + + hbase.coprocessor.master.classes + org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint + + + + hbase.master.loadbalancer.class + org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer + + + +hbase.master.loadbalance.bytabletrue + + + diff --git a/TSG发布版本更新记录/TSG-23.10/hos/galaxy-hos-service.yml b/TSG发布版本更新记录/TSG-23.10/hos/galaxy-hos-service.yml new file mode 100644 index 0000000..c6ad943 --- /dev/null +++ b/TSG发布版本更新记录/TSG-23.10/hos/galaxy-hos-service.yml @@ -0,0 +1,120 @@ +server: + port: 8186 + tomcat: + max-threads: 400 +#tomcat缓存大小,单位KB系统默认10M,配置10g +tomcat: + cacheMaxSize: 1000000 +#hbase参数 +hbase: + zookeeper_quorum: 192.168.44.11,192.168.44.14,192.168.44.15 + zookeeper_property_clientPort: 2181 + zookeeper_znode_parent: /hbase + client_retries_number: 9 + rpc_timeout: 100000 + connect_pool: 10 + client_write_buffer: 10485760 + client_keyvalue_maxsize: 1024000000 + #批量获取数量 + get_batch: 10000 + #part 最大数据量 + maxParts: 100000 + #每次获取的part数 + get_part_batch: 1000 + #每次追加是否更新主文件 + isUpdate: 1 + #hbase索引表前缀,前缀为以下的都为索引表 + time_index_table_prefix: index_time_ + filename_index_table_prefix: index_filename_ + partfile_index_table_prefix: index_partfile_ + system_bucket_meta: system:bucket_meta + #创建表预分区时的分区,为空则不分区 + region_start_key: 1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + filename_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + part_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + #获取文件大小的目录 + hbasePath: /hbase/hbase-2.2.3 + #1是集群0是单机,主要针对存储配额获取方式 + standone: 1 + #hadoop集群namenode节点 + namenodes: 192.168.44.11,192.168.44.14 + #hadoop端口 + hadoop_port: 9000 + hadoop_user: root + hadoop_defaultFS: hdfs://ns1 + hadoop_nameservices: ns1 + hadoop_namenodes_ns1: nn1,nn2 + hadoop_replication: 2 + #建表时是否打开hbase wal,1打开,0关闭 + openWal: 0 + #ttl相关参数 + ttl_scan_batch: 500 + ttl_scan_caching: 1000 + ttl_delete_batch: 1000 + +#是否打开验证,0打开,打开需要使用S3身份验证或者token访问服务 +auth: + open: 0 + #http访问使用的token + token: ENC(vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo) + #s3验证 + s3: + accessKey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) + secretKey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) +hos: + #长连接超时时间 + keepAliveTimeout: 60000 + #批量删除对象的最大数量 + deleteMultipleNumber: 1000 + #获取对象列表等操作的最大值 + maxResultLimit: 100000 + #分块上传的最大分块数 + maxPartNumber: 10000 + #追加上传的最大次数 + maxPosition: 100000 + #存放对象的用户自定义元数据的请求头 + metaHeader: x-hos-meta-message + #存放对象信息的请求头 + objectInfo: x-hos-object-info + #是否快速下载文件,1打开,hbase内存小于20G的集群设为0 + isQuickDownloadFile: 0 + #是否打开对象列表查询功能,1打开 + simple: 1 + #用户白名单(hbase的namespace),获取存储配额 + users: default + #元数据存储占比 + metaProportion: 0.03 + #是否打开限流,0:关闭,1:打开 + openRateLimiter: 1 + #限流每秒请求数 + rateLimiterQps: 20000 + #展示追加文件丢失块的最大数量 + lostPartsCount: 10 + #执行ttl的线程数 + thread: 10 + #是否打开手动ttl,1打开,默认为1 + manualTtl: 0 + #文件最大值 + maxFileSize: 5368709000 + #小文件阈值 + uploadThreshold: 10485760 +#设置上传文件大小的最大值 +spring: + servlet: + multipart: + max-file-size: 1024MB + max-request-size: 1024MB +#Prometheus参数 + application: + name: HosServiceApplication +#Prometheus参数 +management: + endpoints: + web: + exposure: + include: '*' + metrics: + tags: + application: ${spring.application.name} +logging: + config: ./config/log4j2-dev.xml \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-23.11/hbase/hbase-site.xml b/TSG发布版本更新记录/TSG-23.11/hbase/hbase-site.xml new file mode 100644 index 0000000..c916220 --- /dev/null +++ b/TSG发布版本更新记录/TSG-23.11/hbase/hbase-site.xml @@ -0,0 +1,302 @@ + + + + + + hbase.rootdir + hdfs://ns1/hbase/hbase-2.2.3 + + + + hbase.cluster.distributed + true + + + + hbase.zookeeper.quorum + 192.168.44.11,192.168.44.14,192.168.44.15 + + + + hbase.zookeeper.property.clientPort + 2181 + + + + hbase.master.info.port + 60010 + + + + hbase.server.keyvalue.maxsize + 1073741824 + + + + zookeeper.znode.parent + /hbase + + + + hbase.cluster.distributed + true + + + + hbase.rpc.timeout + 300000 + This is for the RPC layer to define how long (millisecond) HBase client applications + take for a remote call to time out. It uses pings to check connections + but will eventually throw a TimeoutException. + + + + zookeeper.session.timeout + 300000 + + + + hbase.regionserver.maxlogs + 64 + + + + + hbase.mob.compaction.mergeable.threshold + 1342177280 + + + + hbase.mob.file.cache.size + 10000 + + Number of opened file handlers to cache. + A larger value will benefit reads by providing more file handlers per mob + file cache and would reduce frequent file opening and closing. + However, if this is set too high, this could lead to a "too many opened file handers" + The default value is 1000. + + + + + + hbase.mob.cache.evict.period + 3600 + + The amount of time in seconds after which an unused file is evicted from the + MOB cache. The default value is 3600 seconds. + + + + + + hbase.mob.cache.evict.remain.ratio + 0.5f + + A multiplier (between 0.0 and 1.0), which determines how many files remain cached + after the threshold of files that remains cached after a cache eviction occurs + which is triggered by reaching the `hbase.mob.file.cache.size` threshold. + The default value is 0.5f, which means that half the files (the least-recently-used + ones) are evicted. + + + + + + hfile.format.version + 3 + + + + hbase.hregion.memstore.flush.size + 134217728 + + + + hbase.hstore.flusher.count + 4 + + + + hbase.regionserver.flush.check.period + 10000 + + It determines the flushing check period of PeriodicFlusher in regionserver. + If unset, it uses hbase.server.thread.wakefrequency as default value. + (in milliseconds) + + + + + hbase.regionserver.compaction.check.period + 10000 + + + + hbase.regionserver.global.memstore.size.lower.limit + 0.95 + + + + hbase.regionserver.global.memstore.size + 0.45 + + + + hfile.block.cache.size + 0.3 + + + + hbase.hregion.memstore.block.multiplier + 2 + + + + hbase.ipc.server.max.callqueue.length + 1073741824 + + + + hbase.regionserver.handler.count + 10 + + Count of RPC Listener instances spun up on RegionServers. + Same property is used by the Master for count of master handlers. + + + + + hbase.zookeeper.property.maxClientCnxns + 1000 + + + + hbase.ipc.max.request.size + 1173741824 + + + + hbase.hstore.blockingWaitTime + 20000 + + The time an HRegion will block updates for after hitting the StoreFile + limit defined by hbase.hstore.blockingStoreFiles. + After this time has elapsed, the HRegion will stop blocking updates even + if a compaction has not been completed. + + + + + hbase.hstore.blockingStoreFiles + 100 + + If more than this number of StoreFiles in any one Store + (one StoreFile is written per flush of MemStore) then updates are + blocked for this HRegion until a compaction is completed, or + until hbase.hstore.blockingWaitTime has been exceeded. + + + + +  hbase.hregion.max.filesize +  10737418240‬ + + + + hbase.regionserver.regionSplitLimit + 500 + + + + phoenix.schema.isNamespaceMappingEnabled + true + + + + phoenix.schema.mapSystemTablesToNamespace + true + + + + hbase.coprocessor.master.classes + org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint + + + + hbase.master.loadbalancer.class + org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer + + + +hbase.master.loadbalance.bytabletrue + + + + + hbase.hstore.compactionThreshold + 5 + + + hbase.hstore.compaction.min + 5 + + + + hbase.hstore.compaction.max + 20 + + + hbase.hstore.compaction.min.size + 134217728 + + + hbase.hstore.compaction.max.size + 10737418240 + + + hbase.regionserver.thread.compaction.throttle + 2684354560 + + + hbase.regionserver.thread.compaction.small + 5 + + + hbase.regionserver.thread.compaction.large + 5 + + + hbase.hstore.compaction.kv.max + 10 + + + hbase.mob.compaction.threads.max + 2 + + + hbase.hregion.majorcompaction + 604800000 + + diff --git a/TSG发布版本更新记录/TSG-23.11/hos/galaxy-hos-service.yml b/TSG发布版本更新记录/TSG-23.11/hos/galaxy-hos-service.yml new file mode 100644 index 0000000..fa5274b --- /dev/null +++ b/TSG发布版本更新记录/TSG-23.11/hos/galaxy-hos-service.yml @@ -0,0 +1,107 @@ +#服务端口 +server: + port: 8186 + tomcat: + max-threads: 400 +#tomcat缓存大小,单位KB系统默认10M,配置10g +tomcat: + cacheMaxSize: 1000000 +#hbase参数 +hbase: + zookeeper_quorum: 192.168.44.11,192.168.44.14,192.168.44.15 + zookeeper_port: 2181 + zookeeper_znode_parent: /hbase + client_retries_number: 9 + rpc_timeout: 100000 + connect_pool: 10 + client_write_buffer: 10485760 + client_key_value_maxsize: 1073741824 + mob_threshold: 10485760 + #part 最大数据量 + max_parts: 100000 + #每次获取的part数 + get_part_batch: 10 + #hbase索引表前缀,前缀为以下的都为索引表 + time_index_table_prefix: index_time_ + filename_index_table_prefix: index_filename_ + partfile_index_table_prefix: index_partfile_ + system_bucket_meta: system:bucket_meta + #创建表预分区时的分区,为空则不分区 + region_start_key: 1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + filename_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + part_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + #获取文件大小的目录 + data_path: /hbase + #1是集群0是单机 + standalone: 1 + #hadoop集群namenode节点 + hadoop_name_nodes: 192.168.44.11,192.168.44.14 + #hadoop端口 + hadoop_port: 9000 + hadoop_user: root + hadoop_default_fs: hdfs://ns1 + hadoop_name_services: ns1 + hadoop_name_nodes_ns1: nn1,nn2 + hadoop_replication: 2 + #ttl相关参数 + ttl_scan_batch: 1000 + ttl_scan_caching: 1000 + ttl_delete_batch: 1000 +#是否打开验证,0打开,打开需要使用S3身份验证或者token访问服务 +auth: + open: 0 + #http访问使用的token + token: ENC(vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo) + #s3验证 + s3: + accesskey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) + secretkey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) +hos: + #文件大小阈值 + maxFileSize: 5073741800 + #大文件阈值 + uploadThreshold: 104857600 + #长连接超时时间 + keepAliveTimeout: 60000 + #批量删除对象的最大数量 + deleteMultipleNumber: 1000 + #获取对象列表等操作的最大值 + maxResultLimit: 100000 + #分块上传的最大分块数 + maxPartNumber: 10000 + #追加上传的最大次数 + maxAppendNumber: 100000 + #是否快速下载文件,1打开,hbase内存小于20G的集群设为0 + isQuickDownloadFile: 0 + #用户白名单(hbase的namespace),获取存储配额 + users: default + #元数据存储占比 + metaProportion: 0.02 + #是否打开限流,0:关闭,1:打开 + openRateLimiter: 0 + #限流每秒请求数 + rateLimiterQps: 20000 + #是否打开手动ttl,1打开,默认为1 + manualTtl: 1 + #执行ttl的线程数 + ttlThread: 10 +#设置上传文件大小的最大值 +spring: + servlet: + multipart: + max-file-size: 1024MB + max-request-size: 1024MB +#Prometheus参数 + application: + name: HosServiceApplication +#Prometheus参数 +management: + endpoints: + web: + exposure: + include: '*' + metrics: + tags: + application: ${spring.application.name} +logging: + config: ./config/log4j2-dev.xml \ No newline at end of file