From e625c5f0dcf9a29c48509d9d051c5a7c6e92d5b0 Mon Sep 17 00:00:00 2001 From: houjinchuan Date: Fri, 15 Dec 2023 16:28:24 +0800 Subject: [PATCH] =?UTF-8?q?23.12=20=E5=A2=9E=E5=8A=A0hbase=E3=80=81hos?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6=EF=BC=8C=E4=BF=AE=E6=94=B9?= =?UTF-8?q?hbase=E3=80=81phoenix=E5=BB=BA=E8=A1=A8=E8=AF=AD=E5=8F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HBase最新建表语句/HBase建表语句_TSG | 8 -- HBase最新建表语句/Phoenix建表语句_TSG.sql | 28 ------- .../TSG-23.11/hbase/hbase-site.xml | 73 +++++------------ .../TSG-23.11/hos/galaxy-hos-service.yml | 65 +++++++++------- .../{TSG-23.10 => TSG-23.12}/hbase/hbase-site.xml | 78 +++++++++++++------ .../hos/galaxy-hos-service.yml | 71 +++++++---------- 6 files changed, 145 insertions(+), 178 deletions(-) rename TSG发布版本更新记录/{TSG-23.10 => TSG-23.12}/hbase/hbase-site.xml (83%) rename TSG发布版本更新记录/{TSG-23.10 => TSG-23.12}/hos/galaxy-hos-service.yml (65%) diff --git a/HBase最新建表语句/HBase建表语句_TSG b/HBase最新建表语句/HBase建表语句_TSG index 692edc2..574f831 100644 --- a/HBase最新建表语句/HBase建表语句_TSG +++ b/HBase最新建表语句/HBase建表语句_TSG @@ -5,11 +5,3 @@ create_namespace 'tsg_galaxy' create 'tsg:report_result', {NAME => 'response', VERSIONS => 1,COMPRESSION => 'GZ',IS_MOB => true, MOB_THRESHOLD => 0}, {NAME => 'detail',COMPRESSION => 'GZ',VERSIONS => 1} create 'dos:ddos_traffic_baselines', 'TCP SYN Flood','UDP Flood','ICMP Flood','DNS Flood' - -create 'tsg_galaxy:relation_account_framedip', {NAME => 'radius', VERSIONS => 1,TTL=> '2592000'}, {NAME => 'common', VERSIONS => 1,TTL=> '2592000'} -create 'tsg_galaxy:relation_framedip_account', {NAME => 'radius', VERSIONS => 1,TTL=> '2592000'}, {NAME => 'common', VERSIONS => 1,TTL=> '2592000'} - -create 'tsg_galaxy:recommendation_app_cip', {NAME => 'common', VERSIONS => 1} - -create 'tsg_galaxy:relation_user_teid',{NAME=>'gtp',TTL=> '604800'}, {NAME => 'common',TTL=> '604800'} -create 'tsg_galaxy:gtpc_knowledge_base',{NAME => 'gtp',TTL=> '604800'}, {NAME => 'common',TTL=> '604800'},SPLITS => ['1','2','3'] \ No newline at end of file diff --git a/HBase最新建表语句/Phoenix建表语句_TSG.sql b/HBase最新建表语句/Phoenix建表语句_TSG.sql index 22f38c9..19a0244 100644 --- a/HBase最新建表语句/Phoenix建表语句_TSG.sql +++ b/HBase最新建表语句/Phoenix建表语句_TSG.sql @@ -3,34 +3,6 @@ CREATE schema IF NOT EXISTS "tsg"; CREATE view "tsg"."report_result"( ROWKEY VARCHAR PRIMARY KEY, "detail"."excute_sql" VARCHAR, "detail"."read_rows" UNSIGNED_LONG, "detail"."result_id" UNSIGNED_INT, "response"."result" VARCHAR); -CREATE view IF NOT EXISTS "tsg_galaxy"."relation_account_framedip"( -ROWKEY VARCHAR PRIMARY KEY, -"common"."vsys_id" UNSIGNED_INT, -"radius"."account" VARCHAR, -"radius"."framed_ip" VARCHAR, -"radius"."first_found_time" UNSIGNED_LONG, -"radius"."last_update_time" UNSIGNED_LONG, -"radius"."acct_status_type" UNSIGNED_INT); - -CREATE view "tsg_galaxy"."recommendation_app_cip"( -ROWKEY VARCHAR PRIMARY KEY, -"common"."app_label" VARCHAR, -"common"."client_ip_list" VARCHAR, -"common"."last_update_time" UNSIGNED_LONG); - -CREATE view IF NOT EXISTS "tsg_galaxy"."gtpc_knowledge_base"( -ROWKEY VARCHAR PRIMARY KEY, -"common"."vsys_id" UNSIGNED_INT, -"gtp"."teid" UNSIGNED_LONG, -"gtp"."uplink_teid" UNSIGNED_LONG, -"gtp"."downlink_teid" UNSIGNED_LONG, -"gtp"."apn" VARCHAR, -"gtp"."phone_number" VARCHAR, -"gtp"."imsi" VARCHAR, -"gtp"."imei" VARCHAR, -"gtp"."msg_type" UNSIGNED_INT, -"gtp"."last_update_time" UNSIGNED_LONG); - CREATE table IF NOT EXISTS "tsg_galaxy"."job_result"( ROWKEY VARCHAR PRIMARY KEY, "detail"."is_error" BOOLEAN, diff --git a/TSG发布版本更新记录/TSG-23.11/hbase/hbase-site.xml b/TSG发布版本更新记录/TSG-23.11/hbase/hbase-site.xml index c916220..cc1dacc 100644 --- a/TSG发布版本更新记录/TSG-23.11/hbase/hbase-site.xml +++ b/TSG发布版本更新记录/TSG-23.11/hbase/hbase-site.xml @@ -48,7 +48,7 @@ hbase.server.keyvalue.maxsize - 1073741824 + 5368709120 @@ -128,12 +128,16 @@ hbase.hregion.memstore.flush.size - 134217728 + 33554432 + + memstore的大小超过该限制（单位：byte）后将被flush到磁盘。这个大小由一个线程间断性的检查，检查的间隔由 + hbase.server.thread.wakefrequency决定 + hbase.hstore.flusher.count - 4 + 2 @@ -164,6 +168,12 @@ hfile.block.cache.size 0.3 + + Percentage of maximum heap (-Xmx setting) to allocate to block cache + used by a StoreFile. Default of 0.4 means allocate 40%. + Set to 0 to disable but it's not recommended; you need at least + enough cache to hold the storefile indices. + @@ -197,7 +207,7 @@ hbase.hstore.blockingWaitTime - 20000 + 30000 The time an HRegion will block updates for after hitting the StoreFile limit defined by hbase.hstore.blockingStoreFiles. @@ -219,12 +229,17 @@ 　 hbase.hregion.max.filesize -　 10737418240‬ +　 3221225472 hbase.regionserver.regionSplitLimit - 500 + 1000 + + + + hbase.replication + true @@ -253,50 +268,4 @@   true - - - hbase.hstore.compactionThreshold - 5 - - - hbase.hstore.compaction.min - 5 - - - - hbase.hstore.compaction.max - 20 - - - hbase.hstore.compaction.min.size - 134217728 - - - hbase.hstore.compaction.max.size - 10737418240 - - - hbase.regionserver.thread.compaction.throttle - 2684354560 - - - hbase.regionserver.thread.compaction.small - 5 - - - hbase.regionserver.thread.compaction.large - 5 - - - hbase.hstore.compaction.kv.max - 10 - - - hbase.mob.compaction.threads.max - 2 - - - hbase.hregion.majorcompaction - 604800000 - diff --git a/TSG发布版本更新记录/TSG-23.11/hos/galaxy-hos-service.yml b/TSG发布版本更新记录/TSG-23.11/hos/galaxy-hos-service.yml index fa5274b..c6ad943 100644 --- a/TSG发布版本更新记录/TSG-23.11/hos/galaxy-hos-service.yml +++ b/TSG发布版本更新记录/TSG-23.11/hos/galaxy-hos-service.yml @@ -1,4 +1,3 @@ -#服务端口 server: port: 8186 tomcat: @@ -9,18 +8,21 @@ tomcat: #hbase参数 hbase: zookeeper_quorum: 192.168.44.11,192.168.44.14,192.168.44.15 - zookeeper_port: 2181 + zookeeper_property_clientPort: 2181 zookeeper_znode_parent: /hbase client_retries_number: 9 rpc_timeout: 100000 connect_pool: 10 client_write_buffer: 10485760 - client_key_value_maxsize: 1073741824 - mob_threshold: 10485760 + client_keyvalue_maxsize: 1024000000 + #批量获取数量 + get_batch: 10000 #part 最大数据量 - max_parts: 100000 + maxParts: 100000 #每次获取的part数 - get_part_batch: 10 + get_part_batch: 1000 + #每次追加是否更新主文件 + isUpdate: 1 #hbase索引表前缀，前缀为以下的都为索引表 time_index_table_prefix: index_time_ filename_index_table_prefix: index_filename_ @@ -31,22 +33,25 @@ hbase: filename_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f part_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f #获取文件大小的目录 - data_path: /hbase - #1是集群0是单机 - standalone: 1 + hbasePath: /hbase/hbase-2.2.3 + #1是集群0是单机，主要针对存储配额获取方式 + standone: 1 #hadoop集群namenode节点 - hadoop_name_nodes: 192.168.44.11,192.168.44.14 + namenodes: 192.168.44.11,192.168.44.14 #hadoop端口 hadoop_port: 9000 hadoop_user: root - hadoop_default_fs: hdfs://ns1 - hadoop_name_services: ns1 - hadoop_name_nodes_ns1: nn1,nn2 + hadoop_defaultFS: hdfs://ns1 + hadoop_nameservices: ns1 + hadoop_namenodes_ns1: nn1,nn2 hadoop_replication: 2 + #建表时是否打开hbase wal，1打开，0关闭 + openWal: 0 #ttl相关参数 - ttl_scan_batch: 1000 + ttl_scan_batch: 500 ttl_scan_caching: 1000 ttl_delete_batch: 1000 + #是否打开验证，0打开，打开需要使用S3身份验证或者token访问服务 auth: open: 0 @@ -54,13 +59,9 @@ auth: token: ENC(vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo) #s3验证 s3: - accesskey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) - secretkey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) + accessKey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) + secretKey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) hos: - #文件大小阈值 - maxFileSize: 5073741800 - #大文件阈值 - uploadThreshold: 104857600 #长连接超时时间 keepAliveTimeout: 60000 #批量删除对象的最大数量 @@ -70,21 +71,33 @@ hos: #分块上传的最大分块数 maxPartNumber: 10000 #追加上传的最大次数 - maxAppendNumber: 100000 + maxPosition: 100000 + #存放对象的用户自定义元数据的请求头 + metaHeader: x-hos-meta-message + #存放对象信息的请求头 + objectInfo: x-hos-object-info #是否快速下载文件，1打开，hbase内存小于20G的集群设为0 isQuickDownloadFile: 0 + #是否打开对象列表查询功能，1打开 + simple: 1 #用户白名单（hbase的namespace），获取存储配额 users: default #元数据存储占比 - metaProportion: 0.02 + metaProportion: 0.03 #是否打开限流,0:关闭，1:打开 - openRateLimiter: 0 + openRateLimiter: 1 #限流每秒请求数 rateLimiterQps: 20000 - #是否打开手动ttl，1打开，默认为1 - manualTtl: 1 + #展示追加文件丢失块的最大数量 + lostPartsCount: 10 #执行ttl的线程数 - ttlThread: 10 + thread: 10 + #是否打开手动ttl，1打开，默认为1 + manualTtl: 0 + #文件最大值 + maxFileSize: 5368709000 + #小文件阈值 + uploadThreshold: 10485760 #设置上传文件大小的最大值 spring: servlet: diff --git a/TSG发布版本更新记录/TSG-23.10/hbase/hbase-site.xml b/TSG发布版本更新记录/TSG-23.12/hbase/hbase-site.xml similarity index 83% rename from TSG发布版本更新记录/TSG-23.10/hbase/hbase-site.xml rename to TSG发布版本更新记录/TSG-23.12/hbase/hbase-site.xml index cc1dacc..d8e9ea9 100644 --- a/TSG发布版本更新记录/TSG-23.10/hbase/hbase-site.xml +++ b/TSG发布版本更新记录/TSG-23.12/hbase/hbase-site.xml @@ -87,7 +87,7 @@ hbase.mob.file.cache.size - 10000 + 5000 Number of opened file handlers to cache. A larger value will benefit reads by providing more file handlers per mob @@ -128,16 +128,12 @@ hbase.hregion.memstore.flush.size - 33554432 - - memstore的大小超过该限制（单位：byte）后将被flush到磁盘。这个大小由一个线程间断性的检查，检查的间隔由 - hbase.server.thread.wakefrequency决定 - + 134217728 hbase.hstore.flusher.count - 2 + 4 @@ -168,21 +164,30 @@ hfile.block.cache.size 0.3 - - Percentage of maximum heap (-Xmx setting) to allocate to block cache - used by a StoreFile. Default of 0.4 means allocate 40%. - Set to 0 to disable but it's not recommended; you need at least - enough cache to hold the storefile indices. - + + + hbase.bucketcache.ioengine + offheap + + + + hbase.bucketcache.size + 512 + hbase.hregion.memstore.block.multiplier - 2 + 4 hbase.ipc.server.max.callqueue.length + 10000 + + + + hbase.ipc.server.max.callqueue.size 1073741824 @@ -207,7 +212,7 @@ hbase.hstore.blockingWaitTime - 30000 + 20000 The time an HRegion will block updates for after hitting the StoreFile limit defined by hbase.hstore.blockingStoreFiles. @@ -229,17 +234,12 @@ 　 hbase.hregion.max.filesize -　 3221225472 +　 10737418240‬ hbase.regionserver.regionSplitLimit - 1000 - - - - hbase.replication - true + 500 @@ -268,4 +268,38 @@   true + + + hbase.hstore.compactionThreshold + 5 + + + hbase.hstore.compaction.min + 5 + + + + hbase.hstore.compaction.max + 20 + + + hbase.hstore.compaction.min.size + 134217728 + + + hbase.hstore.compaction.max.size + 10737418240 + + + hbase.regionserver.thread.compaction.small + 5 + + + hbase.regionserver.thread.compaction.large + 5 + + + hbase.hregion.majorcompaction + 604800000 + diff --git a/TSG发布版本更新记录/TSG-23.10/hos/galaxy-hos-service.yml b/TSG发布版本更新记录/TSG-23.12/hos/galaxy-hos-service.yml similarity index 65% rename from TSG发布版本更新记录/TSG-23.10/hos/galaxy-hos-service.yml rename to TSG发布版本更新记录/TSG-23.12/hos/galaxy-hos-service.yml index c6ad943..053275f 100644 --- a/TSG发布版本更新记录/TSG-23.10/hos/galaxy-hos-service.yml +++ b/TSG发布版本更新记录/TSG-23.12/hos/galaxy-hos-service.yml @@ -1,3 +1,4 @@ +#服务端口 server: port: 8186 tomcat: @@ -8,21 +9,18 @@ tomcat: #hbase参数 hbase: zookeeper_quorum: 192.168.44.11,192.168.44.14,192.168.44.15 - zookeeper_property_clientPort: 2181 + zookeeper_port: 2181 zookeeper_znode_parent: /hbase client_retries_number: 9 rpc_timeout: 100000 connect_pool: 10 client_write_buffer: 10485760 - client_keyvalue_maxsize: 1024000000 - #批量获取数量 - get_batch: 10000 - #part 最大数据量 - maxParts: 100000 + client_key_value_maxsize: 1073741824 + mob_threshold: 10485760 + #part的最大数量 + max_parts: 100000 #每次获取的part数 - get_part_batch: 1000 - #每次追加是否更新主文件 - isUpdate: 1 + get_part_batch: 10 #hbase索引表前缀，前缀为以下的都为索引表 time_index_table_prefix: index_time_ filename_index_table_prefix: index_filename_ @@ -33,25 +31,22 @@ hbase: filename_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f part_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f #获取文件大小的目录 - hbasePath: /hbase/hbase-2.2.3 - #1是集群0是单机，主要针对存储配额获取方式 - standone: 1 + data_path: /hbase + #1是集群0是单机 + standalone: 1 #hadoop集群namenode节点 - namenodes: 192.168.44.11,192.168.44.14 + hadoop_name_nodes: 192.168.44.11,192.168.44.14 #hadoop端口 hadoop_port: 9000 hadoop_user: root - hadoop_defaultFS: hdfs://ns1 - hadoop_nameservices: ns1 - hadoop_namenodes_ns1: nn1,nn2 + hadoop_default_fs: hdfs://ns1 + hadoop_name_services: ns1 + hadoop_name_nodes_ns1: nn1,nn2 hadoop_replication: 2 - #建表时是否打开hbase wal，1打开，0关闭 - openWal: 0 #ttl相关参数 - ttl_scan_batch: 500 + ttl_scan_batch: 1000 ttl_scan_caching: 1000 ttl_delete_batch: 1000 - #是否打开验证，0打开，打开需要使用S3身份验证或者token访问服务 auth: open: 0 @@ -59,9 +54,13 @@ auth: token: ENC(vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo) #s3验证 s3: - accessKey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) - secretKey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) + accesskey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) + secretkey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) hos: + #文件大小阈值 + maxFileSize: 5073741800 + #大文件阈值 + uploadThreshold: 104857600 #长连接超时时间 keepAliveTimeout: 60000 #批量删除对象的最大数量 @@ -71,39 +70,27 @@ hos: #分块上传的最大分块数 maxPartNumber: 10000 #追加上传的最大次数 - maxPosition: 100000 - #存放对象的用户自定义元数据的请求头 - metaHeader: x-hos-meta-message - #存放对象信息的请求头 - objectInfo: x-hos-object-info + maxAppendNumber: 100000 #是否快速下载文件，1打开，hbase内存小于20G的集群设为0 isQuickDownloadFile: 0 - #是否打开对象列表查询功能，1打开 - simple: 1 #用户白名单（hbase的namespace），获取存储配额 users: default #元数据存储占比 - metaProportion: 0.03 + metaProportion: 0.01 #是否打开限流,0:关闭，1:打开 - openRateLimiter: 1 + openRateLimiter: 0 #限流每秒请求数 rateLimiterQps: 20000 - #展示追加文件丢失块的最大数量 - lostPartsCount: 10 - #执行ttl的线程数 - thread: 10 #是否打开手动ttl，1打开，默认为1 - manualTtl: 0 - #文件最大值 - maxFileSize: 5368709000 - #小文件阈值 - uploadThreshold: 10485760 + manualTtl: 1 + #执行ttl的线程数 + ttlThread: 10 #设置上传文件大小的最大值 spring: servlet: multipart: - max-file-size: 1024MB - max-request-size: 1024MB + max-file-size: 5GB + max-request-size: 5GB #Prometheus参数 application: name: HosServiceApplication