diff --git a/tsg_olap/installation/hos/tsg_olap_hos_ddl.sh b/tsg_olap/installation/hos/tsg_olap_hos_ddl.sh index 2c3c15c..ad37488 100644 --- a/tsg_olap/installation/hos/tsg_olap_hos_ddl.sh +++ b/tsg_olap/installation/hos/tsg_olap_hos_ddl.sh @@ -1,5 +1,8 @@ -curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffc_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/troubleshooting_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/knowledge_base_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/report_snapshot_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/assessment_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_policy_capture_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_rtp_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_http_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_eml_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/troubleshooting_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/assessment_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/knowledge_base_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:2' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/report_snapshot_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:2' \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/hbase/update_hbase_24.09.sh b/tsg_olap/upgrade/TSG-24.09/hbase/update_hbase_24.09.sh new file mode 100644 index 0000000..706f127 --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/hbase/update_hbase_24.09.sh @@ -0,0 +1,19 @@ +alter 'troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_time_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_filename_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_partfile_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} + +alter 'assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_time_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_filename_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_partfile_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} + +alter 'knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} +alter 'index_time_knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} +alter 'index_filename_knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} +alter 'index_partfile_knowledge_base_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} + +alter 'report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} +alter 'index_time_report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} +alter 'index_filename_report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} +alter 'index_partfile_report_snapshot_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '2'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '2'}} \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/hos/create_bucket_24.09.sh b/tsg_olap/upgrade/TSG-24.09/hos/create_bucket_24.09.sh new file mode 100644 index 0000000..ad37488 --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/hos/create_bucket_24.09.sh @@ -0,0 +1,8 @@ +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_policy_capture_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_rtp_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_http_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_eml_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/troubleshooting_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/assessment_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/knowledge_base_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:2' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/report_snapshot_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:2' \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/hos/galaxy-hos-service-24.09.yml b/tsg_olap/upgrade/TSG-24.09/hos/galaxy-hos-service-24.09.yml new file mode 100644 index 0000000..b4ce78f --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/hos/galaxy-hos-service-24.09.yml @@ -0,0 +1,97 @@ +#服务端口 +server: + port: 8186 + max-http-header-size: 20MB + tomcat: + max-threads: 400 +#tomcat缓存大小,单位KB系统默认10M,配置10g +tomcat: + cacheMaxSize: 1000000 +#hbase参数 +hbase: + zookeeperQuorum: 192.168.44.11:2181,192.168.44.14:2181,192.168.44.15:2181 + zookeeperPort: 2181 + zookeeperNodeParent: /hbase + clientRetriesNumber: 9 + rpcTimeout: 100000 + connectPool: 10 + clientWriteBuffer: 10485760 + clientKeyValueMaxsize: 1073741824 + mobThreshold: 10485760 + #part的最大数量 + maxParts: 100000 + #每次获取的part数 + getPartBatch: 10 + #hbase索引表前缀,前缀为以下的都为索引表 + timeIndexTablePrefix: index_time_ + filenameIndexTablePrefix: index_filename_ + partFileIndexTablePrefix: index_partfile_ + systemBucketMeta: system:bucket_meta + #创建表的分区数 + regionCount: 16 + filenameHead: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + partHead: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + #获取文件大小的目录 + dataPath: /hbase + #hadoop集群namenode节点,单机为单个ip,集群为ip1,ip2 + hadoopNameNodes: 192.168.44.10,192.168.44.11 + #副本数,单机为1,集群为2 + hadoopReplication: 2 + #hadoop端口 + hadoopPort: 9000 + hadoopUser: root + hadoopNameServices: ns1 + hadoopNameNodesNs1: nn1,nn2 + asyncPut: 0 +#是否打开验证,0打开,打开需要使用S3身份验证或者token访问服务 +auth: + open: 0 + #http访问使用的token + token: ENC(vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo) + #s3验证 + s3: + accesskey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) + secretkey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) +hos: + #文件大小阈值 + maxFileSize: 5073741800 + #大文件阈值 + uploadThreshold: 104857600 + #长连接超时时间 + keepAliveTimeout: 60000 + #批量删除对象的最大数量 + deleteMultipleNumber: 1000 + #获取对象列表等操作的最大值 + maxResultLimit: 100000 + #分块上传的最大分块数 + maxPartNumber: 10000 + #追加上传的最大次数 + maxAppendNumber: 100000 + #是否快速上传 + isQuickUpload: 0 + #是否快速下载文件,1打开,hbase内存小于20G的集群设为0 + isQuickDownloadFile: 0 + #用户白名单(hbase的namespace),获取存储配额 + users: default + #是否打开限流,0:关闭,1:打开 + openRateLimiter: 0 + #限流每秒请求数 + rateLimiterQps: 20000 +#设置上传文件大小的最大值 +spring: + servlet: + multipart: + max-file-size: 5GB + max-request-size: 5GB +#Prometheus参数 + application: + name: HosServiceApplication +#Prometheus参数 +management: + endpoints: + web: + exposure: + include: '*' + metrics: + tags: + application: ${spring.application.name} \ No newline at end of file diff --git a/tsg_olap/upgrade/TSG-24.09/hos/update_hos_24.09.sh b/tsg_olap/upgrade/TSG-24.09/hos/update_hos_24.09.sh new file mode 100644 index 0000000..740b4da --- /dev/null +++ b/tsg_olap/upgrade/TSG-24.09/hos/update_hos_24.09.sh @@ -0,0 +1,6 @@ +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_policy_capture_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_rtp_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_http_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_eml_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' + +curl -X DELETE http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_file_bucket -H 'token:{{ hos_token }}' \ No newline at end of file