diff --git a/galaxy-data-platform/configuration/config.yml b/galaxy-data-platform/configuration/config.yml new file mode 100644 index 0000000..7e4d29c --- /dev/null +++ b/galaxy-data-platform/configuration/config.yml @@ -0,0 +1,30 @@ +#============================Galaxy OLAP Configuration===============================# +#The default installation location +deploy_dir: /data/hdd/olap + +#The default data storage location,use storing application data,logs and configuration files +data_dir: /data/sdd/olap + +#Use commas (,) to separate the network segments that the firewall allows to access +allowed_ips: 192.168.44.0/24,192.168.45.0/24,192.168.58.0/24,192.168.41.0/24 + +#============================Keepalived Configuration===============================# +#Specify the interface virtual IP address.It used to configure high availability for OLAP query engine +#Only one IP address can be configured on the same network segment +#The following three ip addresses must be unique +vrrp_instance: + default: + virtual_ipaddress: 192.168.45.102 + interface: eth0 + virtual_router_id: 61 + oss: + virtual_ipaddress: 192.168.45.102 + interface: eth0 + virtual_router_id: 62 + +#============================OSS Configuration===============================# +#HOS token +hos_server_token: fea0ee76be8147bd9b53ad995b7ef603 + +#HOS token（加密） +hos_servrt_token_encrypt: M+0IMYS1+XENltUe585ahhqwY+QZJmnvtxML1vXExB9aO+CPT3GfsP4rtbVuWXpf diff --git a/galaxy-data-platform/configuration/galaxy-data-platform.yml b/galaxy-data-platform/configuration/galaxy-data-platform.yml new file mode 100644 index 0000000..acb672d --- /dev/null +++ b/galaxy-data-platform/configuration/galaxy-data-platform.yml @@ -0,0 +1,143 @@ +zookeeper: + #Running memory of the Zookeeper. + java_opts: -Xmx1024m -Xms1024m + +mariadb: + #Used to cache data and index data from tables in the InnoDB storage engine. + innodb_buffer_pool_size: 2048 + +nacos: + #Running memory of the Nacos. + java_opt: '-Xmx1024m -Xms1024m -Xmn256m' + +druid: + broker: + #Running memory of the Druid-Broker. + java_opts: -Xmx1024m -Xms1024m + #Worker tasks also use off-heap ("direct") memory. Set the amount of direct memory available (-XX:MaxDirectMemorySize) to at least (druid.processing.numThreads + 1) * druid.processing.buffer.sizeBytes + MaxDirectMemorySize: 512m + #This specifies a buffer size (less than 2GiB), for the storage of intermediate results + druid.processing.buffer.sizeBytes: 50000000 + #The number of direct memory buffers available for merging query results. + druid.processing.numMergeBuffers: 4 + #The number of processing threads to have available for parallel processing of segments. + druid.processing.numThreads: 5 + coordinator: + #Running memory of the Druid-Coordinator. + java_opts: -Xmx1024m -Xms1024m + historical: + #Running memory of the Druid-Historical. + java_opts: -Xmx1024m -Xms1024m + #The size of the process's temporary cache data on disk + druid.segmentCache.locations: 300000000000 + #Worker tasks also use off-heap ("direct") memory. Set the amount of direct memory available (-XX:MaxDirectMemorySize) to at least (druid.processing.numThreads + 1) * druid.processing.buffer.sizeBytes + MaxDirectMemorySize: 512m + #This specifies a buffer size (less than 2GiB), for the storage of intermediate results + druid.processing.buffer.sizeBytes: 50000000 + #The number of direct memory buffers available for merging query results. + druid.processing.numMergeBuffers: 4 + #The number of processing threads to have available for parallel processing of segments. + druid.processing.numThreads: 5 + middlemanager: + #Running memory of the Druid-Middlemanager. + java_opts: -Xmx1024m -Xms1024m + druid.indexer.fork.property.druid.processing.numMergeBuffers: 2 + druid.indexer.fork.property.druid.processing.buffer.sizeBytes: 20000000 + druid.indexer.fork.property.druid.processing.numThreads: 1 + +hadoop: + namenode: + #Running memory of the Hadoop Namenode. + java_opt: '-Xmx1024m -Xms1024m' + #The number of Namenode RPC server threads that listen to requests from clients. + dfs.namenode.handler.count: 30 + datanode: + #Running memory of the Hadoop Datanode. + java_opt: '-Xmx1024m -Xms1024m' + #The number of server threads for the datanode. + dfs.datanode.handler.count: 40 + journalnode: + #Running memory of the Hadoop JournalNode. + java_opt: '-Xmx1024m -Xms1024m' + zkfc: + #Running memory of the Hadoop DFSZKFailoverController. + java_opt: '-Xmx1024m -Xms1024m' + yarn: + resourcemanager: + #Running memory of the Hadoop ResourceManager. + java_opt: '-Xmx1024m -Xms1024m' + nodemanager: + #Running memory of the Hadoop NodeManager. + java_opt: '-Xmx1024m -Xms1024m' + #Amount of physical memory, in MB, that can be allocated for containers. + yarn.nodemanager.resource.memory-mb: 16384 + #The maximum allocation for every container request at the RM in MBs. + yarn.scheduler.maximum-allocation-mb: 16384 + #Number of vcores that can be allocated for containers. This is used by the RM scheduler when allocating resources for containers. + yarn.nodemanager.resource.cpu-vcores: 48 + #The maximum allocation for every container request at the RM in terms of virtual CPU cores. + yarn.scheduler.maximum-allocation-vcores: 48 + +flink: + #Total Process Memory size for the JobManager. + jobmanager.memory.process.size: 1024M + #Total Process Memory size for the TaskExecutors. + taskmanager.memory.process.size: 10240M + #This is the size of off-heap memory managed for sorting, hash tables, caching of intermediate results and state backend. + taskmanager.memory.managed.size: 512M + #Framework Off-Heap Memory size for TaskExecutors. This is the size of off-heap memory reserved for TaskExecutor framework + taskmanager.memory.framework.off-heap.size: 128M + #JVM Metaspace Size for the TaskExecutors. + taskmanager.memory.jvm-metaspace.size: 1024M + #Max Network Memory size for TaskExecutors. Network Memory is off-heap memory reserved for ShuffleEnvironment. + taskmanager.memory.network.max: 256M + #The number of parallel operator or user function instances that a single TaskManager can run. + #This value is typically proportional to the number of physical CPU cores that the TaskManager's machine has (e.g., equal to the number of cores, or half the number of cores). + taskmanager.numberOfTaskSlots: 1 + +hbase: + common: + #The HBase resource isolation function is used to group tables for storage. + enable_rsgroup: false + hmaster: + #Running memory of the HBase HMaster. + java_opt: '-Xmx1024m -Xms1024m' + regionserver: + #Running memory of the HBase HRegionserver. + java_opt: '-Xmx1024m -Xms1024m -Xmn128m' + #This defines the number of threads the region server keeps open to serve requests to tables,It should generally be set to (number of cores - 1) + hbase.regionserver.handler.count: 40 + #If any one of a column families' HStoreFiles has grown to exceed this value, the hosting HRegion is split in two. + hbase.hregion.max.filesize: 10737418240 + #Indicates the memory used by all read caches. The value can be the actual memory value, expressed in MB + hbase.bucketcache.size: 100 + +kafka: + #Running memory of the Kafka. + java_opt: '-Xmx1024m -Xms1024m' + #The minimum age of a log file to be eligible for deletion due to age + log.retention.hours: 168 + #A size-based retention policy for logs,unit byte + log.retention.bytes: 10737418240 + +clickhouse: + #Limit on total memory usage. Zero means Unlimited. + max_server_memory_usage: 30000000000 + #Sets the number of threads performing background merges and mutations for tables with MergeTree engines. + background_pool_size: 16 + +hos: + #Running memory of the Kafka. + java_opt: '-Xmx1024m -Xms1024m -Xmn512m' + #Download files quickly,Used for HBase with a memory larger than 20GB.open: 1 , close: 0 + isQuickDownloadFile: 0 + #Whether to enable SSL.open: 1 , close: 0 + enable_ssl: 0 + #nacos contains the name of the namespace where the configuration is stored + nacos.config.namespace: prod + +ignite: + #Running memory of the Nacos. + java_opt: '-Xmx1024m -Xms1024m' + #Setting region max size equal to physical RAM size(5 GB). + maxSize: '#{5L * 1024 * 1024 * 1024}' diff --git a/galaxy-data-platform/configuration/hosts b/galaxy-data-platform/configuration/hosts new file mode 100644 index 0000000..cd8f665 --- /dev/null +++ b/galaxy-data-platform/configuration/hosts @@ -0,0 +1,111 @@ +#============================================================================== +# Basic Components +# +# Orchestration & Coordinator & Configuration & Cold Storage +#============================================================================== + +#The cluster use master-master replication mode,maximum 2 servers. +[mariadb] + +#Apache Zookeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services. +#The cluster mode at least 3 servers,The number of nodes must be odd,Like 3/5 nodes. +[zookeeper] + + +#Alibaba Nacos an easy-to-use dynamic service discovery, configuration and service management platform +#The cluster mode at least 3 servers,Multi-node HA mode. +[nacos] + + +#Apache Hadoop HDFS(Hadoop Distributed File System) +#HDFS is deployed only in cluster mode. +#At least 3 servers,An HDFS cluster consists of two Namenodes and a certain number of Datanodes node. +[hdfs] + +#============================================================================== +# BigData Processing Components +# +# Big data is a term that refers to the massive volume, variety, and velocity of data that is generated from various sources and needs to be stored, processed, and analyzed efficiently. +# The Big Data processing component is used to provide a platform for fast and efficient processing +#============================================================================== + +#Apache Kafka is a distributed event streaming platform,used for high-performance data pipelines, streaming analytics. +#The cluster mode at least 3 servers,By default install CMAK(Management tool) on the first server. +[kafka] + + +#Apache Hadoop Yarn,Includes the Flink/Groot-stream runtime environment. +#Yarn is the resource management and job scheduling technology in the open source Hadoop distributed processing framework. +#The cluster mode at least 3 servers,A Yarn cluster consists of two ResourceManager (RM) and a certain number of NodeManager(NM) node. +[yarn] + + +#============================================================================== +# Analytic Storage Components +# +# This is a data storage solution designed to support large-scale data analysis and data mining workloads. +# The analytic Storage component it offers high performance, scalability, and flexibility to meet the demands of processing vast amounts of structured and unstructured data. +#============================================================================== + +#Apache HBase is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware +#The cluster mode at least 3 servers,A HBase cluster consists of three HMaster and a certain number of HRegionServer node. +[hbase] + + +#Apache Druid is a high performance, real-time analytics database that delivers sub-second queries on streaming and batch data at scale and under load. +#The cluster mode at least 3 servers,A Druid cluster consists of two master/query and a certain number of worker node. +[druid] + + +#Yandex ClickHouse is the fastest and most resource efficient open-source database for real-time apps and analytics. +#The cluster mode at least 3 servers,A Clickhouse cluster consists of two query and a certain number of data node. +[clickhouse] + + +#ArangoDB is a scalable graph database system to drive value from connected data, faster. +#Only support single server deployment. +[arangodb] + + +#Redis is an open source (BSD licensed), in-memory data structure store, used as a database, cache, and message broker. +#Supports single-node and master-slave modes +[redis] + + +#Apache Ignite is a leading distributed database management system for high-performance computing with in-memory speed +#Supports single-node and cluster modes +[ignite] + + +#============================================================================== +# OLAP Self-research service +# +#============================================================================== + +#The default proxy,Includes the Nginx/Keepalived,If it is a standalone mode, only Nginx. +#A maximum of two nodes. +[loadbalancer] + + +#The clickhouse query proxy,Usually deployed with loadbalancer. +[chproxy] + + +#Galaxy-hos-service is a distributed object storage service. +#Include components:Keepalived/Nginx/Galaxy-hos-service,If it is a standalone mode, only Galaxy-hos-service/Nginx. +#The cluster mode at least 2 servers,keepalived and nginx services are deployed on the first two nodes by default. +[galaxy_hos_service] + + +#The query gateway,Provides a unified query entry +[galaxy_qgw_service] + + +#A lightweight distributed task scheduling framework. +#Include components: Galaxy-job-admin/Galaxy-job-executor +[galaxy_job_service] + + +#The report execution service. +[saved_query_scheduler] + diff --git a/Apache Druid/26.0.0/druid/hosts b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/hosts similarity index 100% rename from Apache Druid/26.0.0/druid/hosts rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/hosts diff --git a/Apache Druid/26.0.0/druid/install.yml b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/install.yml similarity index 100% rename from Apache Druid/26.0.0/druid/install.yml rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/install.yml diff --git a/Apache Druid/26.0.0/druid/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/defaults/main.yml similarity index 100% rename from Apache Druid/26.0.0/druid/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/defaults/main.yml diff --git a/Apache Druid/26.0.0/druid/role/files/conf.zip b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/files/conf.zip similarity index 100% rename from Apache Druid/26.0.0/druid/role/files/conf.zip rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/files/conf.zip diff --git a/Apache Druid/26.0.0/druid/role/files/mysql b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/files/mysql similarity index 100% rename from Apache Druid/26.0.0/druid/role/files/mysql rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/files/mysql diff --git a/Apache Druid/26.0.0/druid/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/handlers/main.yml similarity index 100% rename from Apache Druid/26.0.0/druid/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/handlers/main.yml diff --git a/Apache Druid/26.0.0/druid/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/deploy.yml similarity index 100% rename from Apache Druid/26.0.0/druid/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/deploy.yml diff --git a/Apache Druid/26.0.0/druid/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/main.yml similarity index 100% rename from Apache Druid/26.0.0/druid/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/main.yml diff --git a/Apache Druid/26.0.0/druid/role/tasks/standalone/deploy.yml b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/standalone/deploy.yml similarity index 100% rename from Apache Druid/26.0.0/druid/role/tasks/standalone/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/standalone/deploy.yml diff --git a/Apache Druid/26.0.0/druid/role/tasks/standalone/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/standalone/uninstall.yml similarity index 100% rename from Apache Druid/26.0.0/druid/role/tasks/standalone/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/standalone/uninstall.yml diff --git a/Apache Druid/26.0.0/druid/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/status-check.yml similarity index 100% rename from Apache Druid/26.0.0/druid/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/status-check.yml diff --git a/Apache Druid/26.0.0/druid/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/uninstall.yml similarity index 100% rename from Apache Druid/26.0.0/druid/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/tasks/uninstall.yml diff --git a/Apache Druid/26.0.0/druid/role/templates/broker_jvm.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/broker_jvm.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/broker_jvm.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/broker_jvm.j2 diff --git a/Apache Druid/26.0.0/druid/role/templates/broker_runtime.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/broker_runtime.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/broker_runtime.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/broker_runtime.j2 diff --git a/Apache Druid/26.0.0/druid/role/templates/common.runtime.properties.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/common.runtime.properties.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/common.runtime.properties.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/common.runtime.properties.j2 diff --git a/Apache Druid/26.0.0/druid/role/templates/coordinator_jvm.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/coordinator_jvm.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/coordinator_jvm.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/coordinator_jvm.j2 diff --git a/Apache Druid/26.0.0/druid/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/docker-compose.yml.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/docker-compose.yml.j2 diff --git a/Apache Druid/26.0.0/druid/role/templates/docker-compose_exporter.yml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/docker-compose_exporter.yml.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/docker-compose_exporter.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/docker-compose_exporter.yml.j2 diff --git a/Apache Druid/26.0.0/druid/role/templates/historical_jvm.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/historical_jvm.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/historical_jvm.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/historical_jvm.j2 diff --git a/Apache Druid/26.0.0/druid/role/templates/historical_runtime.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/historical_runtime.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/historical_runtime.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/historical_runtime.j2 diff --git a/Apache Druid/26.0.0/druid/role/templates/middleManager_jvm.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/middleManager_jvm.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/middleManager_jvm.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/middleManager_jvm.j2 diff --git a/Apache Druid/26.0.0/druid/role/templates/middleManager_runtime.properties.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/middleManager_runtime.properties.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/middleManager_runtime.properties.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/middleManager_runtime.properties.j2 diff --git a/Apache Druid/26.0.0/druid/role/templates/router_runtime.properties.j2 b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/router_runtime.properties.j2 similarity index 100% rename from Apache Druid/26.0.0/druid/role/templates/router_runtime.properties.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/templates/router_runtime.properties.j2 diff --git a/Apache Druid/26.0.0/druid/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/vars/main.yml similarity index 100% rename from Apache Druid/26.0.0/druid/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Druid/26.0.0/druid/role/vars/main.yml diff --git a/Apache HBase/2.2.3/hbase/hosts b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/hosts similarity index 100% rename from Apache HBase/2.2.3/hbase/hosts rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/hosts diff --git a/Apache HBase/2.2.3/hbase/install.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/install.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/install.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/install.yml diff --git a/Apache HBase/2.2.3/hbase/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/defaults/main.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/defaults/main.yml diff --git a/Apache HBase/2.2.3/hbase/role/files/conf.zip b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/files/conf.zip similarity index 100% rename from Apache HBase/2.2.3/hbase/role/files/conf.zip rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/files/conf.zip diff --git a/Apache HBase/2.2.3/hbase/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/handlers/main.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/handlers/main.yml diff --git a/Apache HBase/2.2.3/hbase/role/tasks/deploy-cluster.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/deploy-cluster.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/role/tasks/deploy-cluster.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/deploy-cluster.yml diff --git a/Apache HBase/2.2.3/hbase/role/tasks/deploy-standalone.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/deploy-standalone.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/role/tasks/deploy-standalone.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/deploy-standalone.yml diff --git a/Apache HBase/2.2.3/hbase/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/main.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/main.yml diff --git a/Apache HBase/2.2.3/hbase/role/tasks/standalone/deploy.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/standalone/deploy.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/role/tasks/standalone/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/standalone/deploy.yml diff --git a/Apache HBase/2.2.3/hbase/role/tasks/standalone/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/standalone/uninstall.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/role/tasks/standalone/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/standalone/uninstall.yml diff --git a/Apache HBase/2.2.3/hbase/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/status-check.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/status-check.yml diff --git a/Apache HBase/2.2.3/hbase/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/uninstall.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/tasks/uninstall.yml diff --git a/Apache HBase/2.2.3/hbase/role/templates/backup-masters.j2 b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/backup-masters.j2 similarity index 100% rename from Apache HBase/2.2.3/hbase/role/templates/backup-masters.j2 rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/backup-masters.j2 diff --git a/Apache HBase/2.2.3/hbase/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/docker-compose.yml.j2 similarity index 100% rename from Apache HBase/2.2.3/hbase/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/docker-compose.yml.j2 diff --git a/Apache HBase/2.2.3/hbase/role/templates/hbase-env.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/hbase-env.sh.j2 similarity index 100% rename from Apache HBase/2.2.3/hbase/role/templates/hbase-env.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/hbase-env.sh.j2 diff --git a/Apache HBase/2.2.3/hbase/role/templates/hbase-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/hbase-site.xml.j2 similarity index 100% rename from Apache HBase/2.2.3/hbase/role/templates/hbase-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/hbase-site.xml.j2 diff --git a/Apache HBase/2.2.3/hbase/role/templates/regionservers.j2 b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/regionservers.j2 similarity index 100% rename from Apache HBase/2.2.3/hbase/role/templates/regionservers.j2 rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/regionservers.j2 diff --git a/Apache HBase/2.2.3/hbase/role/templates/rsgroup.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/rsgroup.sh.j2 similarity index 100% rename from Apache HBase/2.2.3/hbase/role/templates/rsgroup.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/rsgroup.sh.j2 diff --git a/Apache HBase/2.2.3/hbase/role/templates/startsql.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/startsql.sh.j2 similarity index 100% rename from Apache HBase/2.2.3/hbase/role/templates/startsql.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/templates/startsql.sh.j2 diff --git a/Apache HBase/2.2.3/hbase/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/vars/main.yml similarity index 100% rename from Apache HBase/2.2.3/hbase/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache HBase/2.2.3/hbase/role/vars/main.yml diff --git a/Apache Hadoop/2.7.1/hdfs/hosts b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/hosts similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/hosts rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/hosts diff --git a/Apache Hadoop/2.7.1/hdfs/install.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/install.yml similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/install.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/install.yml diff --git a/Apache Hadoop/2.7.1/hdfs/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/defaults/main.yml similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/defaults/main.yml diff --git a/Apache Hadoop/2.7.1/hdfs/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/tasks/deploy.yml similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/tasks/deploy.yml diff --git a/Apache Hadoop/2.7.1/hdfs/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/tasks/main.yml similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/tasks/main.yml diff --git a/Apache Hadoop/2.7.1/hdfs/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/tasks/status-check.yml similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/tasks/status-check.yml diff --git a/Apache Hadoop/2.7.1/hdfs/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/tasks/uninstall.yml similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/tasks/uninstall.yml diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/core-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/core-site.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/core-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/core-site.xml.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsjournal.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsjournal.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsjournal.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsjournal.sh.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsmaster.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsmaster.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsmaster.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsmaster.sh.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsslave.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsslave.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsslave.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsslave.sh.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsworker.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsworker.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsworker.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/dae-hdfsworker.sh.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsjournal.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsjournal.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsjournal.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsjournal.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsmaster.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsmaster.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsmaster.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsmaster.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsslave.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsslave.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsslave.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsslave.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsworker.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsworker.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsworker.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/daemonscript/keephdfsworker.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/hadoop-env.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/hadoop-env.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/hadoop-env.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/hadoop-env.sh.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/hdfs-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/hdfs-site.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/hdfs-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/hdfs-site.xml.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/ini_hdfs.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/ini_hdfs.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/ini_hdfs.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/ini_hdfs.sh.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/mapred-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/mapred-site.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/mapred-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/mapred-site.xml.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/set_hdfs_env.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/set_hdfs_env.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/set_hdfs_env.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/set_hdfs_env.sh.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/slaves.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/slaves.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/slaves.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/slaves.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/templates/unload_hdfs.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/unload_hdfs.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/templates/unload_hdfs.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/templates/unload_hdfs.sh.j2 diff --git a/Apache Hadoop/2.7.1/hdfs/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/vars/main.yml similarity index 100% rename from Apache Hadoop/2.7.1/hdfs/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/hdfs/role/vars/main.yml diff --git a/Apache Hadoop/2.7.1/yarn/hosts b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/hosts similarity index 100% rename from Apache Hadoop/2.7.1/yarn/hosts rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/hosts diff --git a/Apache Hadoop/2.7.1/yarn/install.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/install.yml similarity index 100% rename from Apache Hadoop/2.7.1/yarn/install.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/install.yml diff --git a/Apache Hadoop/2.7.1/yarn/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/defaults/main.yml similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/defaults/main.yml diff --git a/Apache Hadoop/2.7.1/yarn/role/tasks/deploy-cluster.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/tasks/deploy-cluster.yml similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/tasks/deploy-cluster.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/tasks/deploy-cluster.yml diff --git a/Apache Hadoop/2.7.1/yarn/role/tasks/deploy-standalone.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/tasks/deploy-standalone.yml similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/tasks/deploy-standalone.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/tasks/deploy-standalone.yml diff --git a/Apache Hadoop/2.7.1/yarn/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/tasks/main.yml similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/tasks/main.yml diff --git a/Apache Hadoop/2.7.1/yarn/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/tasks/status-check.yml similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/tasks/status-check.yml diff --git a/Apache Hadoop/2.7.1/yarn/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/tasks/uninstall.yml similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/tasks/uninstall.yml diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/capacity-scheduler.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/capacity-scheduler.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/capacity-scheduler.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/capacity-scheduler.xml.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/core-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/core-site.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/core-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/core-site.xml.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnhistory.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnhistory.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnhistory.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnhistory.sh.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnmaster.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnmaster.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnmaster.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnmaster.sh.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnworker.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnworker.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnworker.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/dae-yarnworker.sh.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnhistory.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnhistory.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnhistory.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnhistory.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnmaster.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnmaster.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnmaster.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnmaster.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnworker.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnworker.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnworker.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/daemonscript/keepyarnworker.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/flink/flink-conf.yaml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/flink/flink-conf.yaml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/flink/flink-conf.yaml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/flink/flink-conf.yaml.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/flink/flink.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/flink/flink.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/flink/flink.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/flink/flink.sh.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/hadoop-env.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/hadoop-env.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/hadoop-env.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/hadoop-env.sh.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/hdfs-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/hdfs-site.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/hdfs-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/hdfs-site.xml.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/mapred-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/mapred-site.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/mapred-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/mapred-site.xml.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/set_yarn_env.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/set_yarn_env.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/set_yarn_env.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/set_yarn_env.sh.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/slaves.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/slaves.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/slaves.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/slaves.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/standalone/core-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/standalone/core-site.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/standalone/core-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/standalone/core-site.xml.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/standalone/hdfs-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/standalone/hdfs-site.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/standalone/hdfs-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/standalone/hdfs-site.xml.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/standalone/yarn-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/standalone/yarn-site.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/standalone/yarn-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/standalone/yarn-site.xml.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/unload_hadoop_yarn.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/unload_hadoop_yarn.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/unload_hadoop_yarn.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/unload_hadoop_yarn.sh.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/yarn-env.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/yarn-env.sh.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/yarn-env.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/yarn-env.sh.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/templates/yarn-site.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/yarn-site.xml.j2 similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/templates/yarn-site.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/templates/yarn-site.xml.j2 diff --git a/Apache Hadoop/2.7.1/yarn/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/vars/main.yml similarity index 100% rename from Apache Hadoop/2.7.1/yarn/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Hadoop/2.7.1/yarn/role/vars/main.yml diff --git a/Apache Ignite/2.15.0/ignite/hosts b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/hosts similarity index 100% rename from Apache Ignite/2.15.0/ignite/hosts rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/hosts diff --git a/Apache Ignite/2.15.0/ignite/install.yml b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/install.yml similarity index 100% rename from Apache Ignite/2.15.0/ignite/install.yml rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/install.yml diff --git a/Apache Ignite/2.15.0/ignite/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/defaults/main.yml similarity index 100% rename from Apache Ignite/2.15.0/ignite/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/defaults/main.yml diff --git a/Apache Ignite/2.15.0/ignite/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/handlers/main.yml similarity index 100% rename from Apache Ignite/2.15.0/ignite/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/handlers/main.yml diff --git a/Apache Ignite/2.15.0/ignite/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/tasks/deploy.yml similarity index 100% rename from Apache Ignite/2.15.0/ignite/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/tasks/deploy.yml diff --git a/Apache Ignite/2.15.0/ignite/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/tasks/main.yml similarity index 100% rename from Apache Ignite/2.15.0/ignite/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/tasks/main.yml diff --git a/Apache Ignite/2.15.0/ignite/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/tasks/status-check.yml similarity index 100% rename from Apache Ignite/2.15.0/ignite/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/tasks/status-check.yml diff --git a/Apache Ignite/2.15.0/ignite/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/tasks/uninstall.yml similarity index 100% rename from Apache Ignite/2.15.0/ignite/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/tasks/uninstall.yml diff --git a/Apache Ignite/2.15.0/ignite/role/templates/default-config.xml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/templates/default-config.xml.j2 similarity index 100% rename from Apache Ignite/2.15.0/ignite/role/templates/default-config.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/templates/default-config.xml.j2 diff --git a/Apache Ignite/2.15.0/ignite/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/templates/docker-compose.yml.j2 similarity index 100% rename from Apache Ignite/2.15.0/ignite/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/templates/docker-compose.yml.j2 diff --git a/Apache Ignite/2.15.0/ignite/role/vars/.main.yml.swp b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/vars/.main.yml.swp similarity index 100% rename from Apache Ignite/2.15.0/ignite/role/vars/.main.yml.swp rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/vars/.main.yml.swp diff --git a/Apache Ignite/2.15.0/ignite/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/vars/main.yml similarity index 100% rename from Apache Ignite/2.15.0/ignite/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Ignite/2.15.0/ignite/role/vars/main.yml diff --git a/Apache Kafka/3.4.1/kafka/hosts b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/hosts similarity index 100% rename from Apache Kafka/3.4.1/kafka/hosts rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/hosts diff --git a/Apache Kafka/3.4.1/kafka/install.yml b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/install.yml similarity index 100% rename from Apache Kafka/3.4.1/kafka/install.yml rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/install.yml diff --git a/Apache Kafka/3.4.1/kafka/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/defaults/main.yml similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/defaults/main.yml diff --git a/Apache Kafka/3.4.1/kafka/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/handlers/main.yml similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/handlers/main.yml diff --git a/Apache Kafka/3.4.1/kafka/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/tasks/deploy.yml similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/tasks/deploy.yml diff --git a/Apache Kafka/3.4.1/kafka/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/tasks/main.yml similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/tasks/main.yml diff --git a/Apache Kafka/3.4.1/kafka/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/tasks/status-check.yml similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/tasks/status-check.yml diff --git a/Apache Kafka/3.4.1/kafka/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/tasks/uninstall.yml similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/tasks/uninstall.yml diff --git a/Apache Kafka/3.4.1/kafka/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/templates/docker-compose.yml.j2 similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/templates/docker-compose.yml.j2 diff --git a/Apache Kafka/3.4.1/kafka/role/templates/docker-compose_exporter.yml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/templates/docker-compose_exporter.yml.j2 similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/templates/docker-compose_exporter.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/templates/docker-compose_exporter.yml.j2 diff --git a/Apache Kafka/3.4.1/kafka/role/templates/kafka-operation.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/templates/kafka-operation.sh.j2 similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/templates/kafka-operation.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/templates/kafka-operation.sh.j2 diff --git a/Apache Kafka/3.4.1/kafka/role/templates/kafka.sh.j2 b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/templates/kafka.sh.j2 similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/templates/kafka.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/templates/kafka.sh.j2 diff --git a/Apache Kafka/3.4.1/kafka/role/templates/server.properties.j2 b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/templates/server.properties.j2 similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/templates/server.properties.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/templates/server.properties.j2 diff --git a/Apache Kafka/3.4.1/kafka/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/vars/main.yml similarity index 100% rename from Apache Kafka/3.4.1/kafka/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Kafka/3.4.1/kafka/role/vars/main.yml diff --git a/Apache Zookeeper/3.5.9/zookeeper/hosts b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/hosts similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/hosts rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/hosts diff --git a/Apache Zookeeper/3.5.9/zookeeper/install.yml b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/install.yml similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/install.yml rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/install.yml diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/defaults/main.yml similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/defaults/main.yml diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/handlers/main.yml similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/handlers/main.yml diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/tasks/deploy.yml similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/tasks/deploy.yml diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/tasks/main.yml similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/tasks/main.yml diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/tasks/status-check.yml similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/tasks/status-check.yml diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/tasks/uninstall.yml similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/tasks/uninstall.yml diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/templates/docker-compose.yml.j2 similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/templates/docker-compose.yml.j2 diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/templates/docker-compose_exporter.yml.j2 b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/templates/docker-compose_exporter.yml.j2 similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/templates/docker-compose_exporter.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/templates/docker-compose_exporter.yml.j2 diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/templates/myid.j2 b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/templates/myid.j2 similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/templates/myid.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/templates/myid.j2 diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/templates/zoo.cfg.j2 b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/templates/zoo.cfg.j2 similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/templates/zoo.cfg.j2 rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/templates/zoo.cfg.j2 diff --git a/Apache Zookeeper/3.5.9/zookeeper/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/vars/main.yml similarity index 100% rename from Apache Zookeeper/3.5.9/zookeeper/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Apache Zookeeper/3.5.9/zookeeper/role/vars/main.yml diff --git a/ArangoDB/3.6.4/arangodb/hosts b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/hosts similarity index 100% rename from ArangoDB/3.6.4/arangodb/hosts rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/hosts diff --git a/ArangoDB/3.6.4/arangodb/install.yml b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/install.yml similarity index 100% rename from ArangoDB/3.6.4/arangodb/install.yml rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/install.yml diff --git a/ArangoDB/3.6.4/arangodb/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/defaults/main.yml similarity index 100% rename from ArangoDB/3.6.4/arangodb/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/defaults/main.yml diff --git a/ArangoDB/3.6.4/arangodb/role/files/init.zip b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/files/init.zip similarity index 100% rename from ArangoDB/3.6.4/arangodb/role/files/init.zip rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/files/init.zip diff --git a/ArangoDB/3.6.4/arangodb/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/handlers/main.yml similarity index 100% rename from ArangoDB/3.6.4/arangodb/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/handlers/main.yml diff --git a/ArangoDB/3.6.4/arangodb/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/tasks/deploy.yml similarity index 100% rename from ArangoDB/3.6.4/arangodb/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/tasks/deploy.yml diff --git a/ArangoDB/3.6.4/arangodb/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/tasks/main.yml similarity index 100% rename from ArangoDB/3.6.4/arangodb/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/tasks/main.yml diff --git a/ArangoDB/3.6.4/arangodb/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/tasks/status-check.yml similarity index 100% rename from ArangoDB/3.6.4/arangodb/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/tasks/status-check.yml diff --git a/ArangoDB/3.6.4/arangodb/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/tasks/uninstall.yml similarity index 100% rename from ArangoDB/3.6.4/arangodb/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/tasks/uninstall.yml diff --git a/ArangoDB/3.6.4/arangodb/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/templates/docker-compose.yml.j2 similarity index 100% rename from ArangoDB/3.6.4/arangodb/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/templates/docker-compose.yml.j2 diff --git a/ArangoDB/3.6.4/arangodb/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/vars/main.yml similarity index 100% rename from ArangoDB/3.6.4/arangodb/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/ArangoDB/3.6.4/arangodb/role/vars/main.yml diff --git a/CMAK/3.0.0.6/cmak/hosts b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/hosts similarity index 100% rename from CMAK/3.0.0.6/cmak/hosts rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/hosts diff --git a/CMAK/3.0.0.6/cmak/install.yml b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/install.yml similarity index 100% rename from CMAK/3.0.0.6/cmak/install.yml rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/install.yml diff --git a/CMAK/3.0.0.6/cmak/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/defaults/main.yml similarity index 100% rename from CMAK/3.0.0.6/cmak/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/defaults/main.yml diff --git a/CMAK/3.0.0.6/cmak/role/files/kafka_client_jaas.conf b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/files/kafka_client_jaas.conf similarity index 100% rename from CMAK/3.0.0.6/cmak/role/files/kafka_client_jaas.conf rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/files/kafka_client_jaas.conf diff --git a/CMAK/3.0.0.6/cmak/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/handlers/main.yml similarity index 100% rename from CMAK/3.0.0.6/cmak/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/handlers/main.yml diff --git a/CMAK/3.0.0.6/cmak/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/tasks/deploy.yml similarity index 100% rename from CMAK/3.0.0.6/cmak/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/tasks/deploy.yml diff --git a/CMAK/3.0.0.6/cmak/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/tasks/main.yml similarity index 100% rename from CMAK/3.0.0.6/cmak/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/tasks/main.yml diff --git a/CMAK/3.0.0.6/cmak/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/tasks/status-check.yml similarity index 100% rename from CMAK/3.0.0.6/cmak/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/tasks/status-check.yml diff --git a/CMAK/3.0.0.6/cmak/role/tasks/unload.yml b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/tasks/unload.yml similarity index 100% rename from CMAK/3.0.0.6/cmak/role/tasks/unload.yml rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/tasks/unload.yml diff --git a/CMAK/3.0.0.6/cmak/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/templates/docker-compose.yml.j2 similarity index 100% rename from CMAK/3.0.0.6/cmak/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/templates/docker-compose.yml.j2 diff --git a/CMAK/3.0.0.6/cmak/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/vars/main.yml similarity index 100% rename from CMAK/3.0.0.6/cmak/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/CMAK/3.0.0.6/cmak/role/vars/main.yml diff --git a/Chproxy/21.06.30/chproxy/hosts b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/hosts similarity index 100% rename from Chproxy/21.06.30/chproxy/hosts rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/hosts diff --git a/Chproxy/21.06.30/chproxy/install.yml b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/install.yml similarity index 100% rename from Chproxy/21.06.30/chproxy/install.yml rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/install.yml diff --git a/Chproxy/21.06.30/chproxy/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/defaults/main.yml similarity index 100% rename from Chproxy/21.06.30/chproxy/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/defaults/main.yml diff --git a/Chproxy/21.06.30/chproxy/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/handlers/main.yml similarity index 100% rename from Chproxy/21.06.30/chproxy/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/handlers/main.yml diff --git a/Chproxy/21.06.30/chproxy/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/tasks/deploy.yml similarity index 100% rename from Chproxy/21.06.30/chproxy/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/tasks/deploy.yml diff --git a/Chproxy/21.06.30/chproxy/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/tasks/main.yml similarity index 100% rename from Chproxy/21.06.30/chproxy/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/tasks/main.yml diff --git a/Chproxy/21.06.30/chproxy/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/tasks/status-check.yml similarity index 100% rename from Chproxy/21.06.30/chproxy/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/tasks/status-check.yml diff --git a/Chproxy/21.06.30/chproxy/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/tasks/uninstall.yml similarity index 100% rename from Chproxy/21.06.30/chproxy/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/tasks/uninstall.yml diff --git a/Chproxy/21.06.30/chproxy/role/templates/config.yml.j2 b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/templates/config.yml.j2 similarity index 100% rename from Chproxy/21.06.30/chproxy/role/templates/config.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/templates/config.yml.j2 diff --git a/Chproxy/21.06.30/chproxy/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/templates/docker-compose.yml.j2 similarity index 100% rename from Chproxy/21.06.30/chproxy/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/templates/docker-compose.yml.j2 diff --git a/Chproxy/21.06.30/chproxy/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/vars/main.yml similarity index 100% rename from Chproxy/21.06.30/chproxy/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Chproxy/21.06.30/chproxy/role/vars/main.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/hosts b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/hosts similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/hosts rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/hosts diff --git a/Clickhouse/21.8.13.1/clickhouse/install.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/install.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/install.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/install.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/defaults/main.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/defaults/main.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/handlers/main.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/handlers/main.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/deploy.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/deploy.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/deploy.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/status-check.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/status-check.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/status-check.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/uninstall.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/cluster/uninstall.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/main.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/main.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/deploy.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/deploy.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/deploy.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/status-check.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/status-check.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/status-check.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/uninstall.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/tasks/standalone/uninstall.yml diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/ck_monitor.sh.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/ck_monitor.sh.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/ck_monitor.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/ck_monitor.sh.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/clickhouse-server.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/clickhouse-server.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/clickhouse-server.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/clickhouse-server.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/config.xml.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/config.xml.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/config.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/config.xml.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/docker-compose_exporter.yml.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/docker-compose_exporter.yml.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/docker-compose_exporter.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/docker-compose_exporter.yml.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/metrika_data.xml.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/metrika_data.xml.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/metrika_data.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/metrika_data.xml.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/metrika_query.xml.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/metrika_query.xml.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/metrika_query.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/metrika_query.xml.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/ck_monitor.sh.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/ck_monitor.sh.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/ck_monitor.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/ck_monitor.sh.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/config.xml.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/config.xml.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/config.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/config.xml.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/docker-compose.yml.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/docker-compose.yml.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/metrika_standalone.xml.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/metrika_standalone.xml.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/metrika_standalone.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/standalone/metrika_standalone.xml.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/unload_ck.sh.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/unload_ck.sh.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/unload_ck.sh.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/unload_ck.sh.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/templates/users.xml.j2 b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/users.xml.j2 similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/templates/users.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/templates/users.xml.j2 diff --git a/Clickhouse/21.8.13.1/clickhouse/role/vars/.main.yml.swp b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/vars/.main.yml.swp similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/vars/.main.yml.swp rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/vars/.main.yml.swp diff --git a/Clickhouse/21.8.13.1/clickhouse/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/vars/main.yml similarity index 100% rename from Clickhouse/21.8.13.1/clickhouse/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Clickhouse/21.8.13.1/clickhouse/role/vars/main.yml diff --git a/GrootStream/1.0/grootstream/hosts b/galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/hosts similarity index 100% rename from GrootStream/1.0/grootstream/hosts rename to galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/hosts diff --git a/GrootStream/1.0/grootstream/install.yml b/galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/install.yml similarity index 100% rename from GrootStream/1.0/grootstream/install.yml rename to galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/install.yml diff --git a/GrootStream/1.0/grootstream/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/defaults/main.yml similarity index 100% rename from GrootStream/1.0/grootstream/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/defaults/main.yml diff --git a/GrootStream/1.0/grootstream/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/tasks/deploy.yml similarity index 100% rename from GrootStream/1.0/grootstream/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/tasks/deploy.yml diff --git a/GrootStream/1.0/grootstream/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/tasks/main.yml similarity index 100% rename from GrootStream/1.0/grootstream/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/tasks/main.yml diff --git a/GrootStream/1.0/grootstream/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/tasks/uninstall.yml similarity index 100% rename from GrootStream/1.0/grootstream/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/tasks/uninstall.yml diff --git a/GrootStream/1.0/grootstream/role/templates/groot-stream.sh.j2 b/galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/templates/groot-stream.sh.j2 similarity index 100% rename from GrootStream/1.0/grootstream/role/templates/groot-stream.sh.j2 rename to galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/templates/groot-stream.sh.j2 diff --git a/GrootStream/1.0/grootstream/role/templates/grootstream.yaml.j2 b/galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/templates/grootstream.yaml.j2 similarity index 100% rename from GrootStream/1.0/grootstream/role/templates/grootstream.yaml.j2 rename to galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/templates/grootstream.yaml.j2 diff --git a/GrootStream/1.0/grootstream/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/vars/main.yml similarity index 100% rename from GrootStream/1.0/grootstream/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/GrootStream/1.0/grootstream/role/vars/main.yml diff --git a/MariaDB/10.5.3/mariadb/hosts b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/hosts similarity index 100% rename from MariaDB/10.5.3/mariadb/hosts rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/hosts diff --git a/MariaDB/10.5.3/mariadb/install.yml b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/install.yml similarity index 100% rename from MariaDB/10.5.3/mariadb/install.yml rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/install.yml diff --git a/MariaDB/10.5.3/mariadb/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/defaults/main.yml similarity index 100% rename from MariaDB/10.5.3/mariadb/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/defaults/main.yml diff --git a/MariaDB/10.5.3/mariadb/role/files/pyMysql.zip b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/files/pyMysql.zip similarity index 100% rename from MariaDB/10.5.3/mariadb/role/files/pyMysql.zip rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/files/pyMysql.zip diff --git a/MariaDB/10.5.3/mariadb/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/handlers/main.yml similarity index 100% rename from MariaDB/10.5.3/mariadb/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/handlers/main.yml diff --git a/MariaDB/10.5.3/mariadb/role/tasks/deploy-cluster.yml b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/tasks/deploy-cluster.yml similarity index 100% rename from MariaDB/10.5.3/mariadb/role/tasks/deploy-cluster.yml rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/tasks/deploy-cluster.yml diff --git a/MariaDB/10.5.3/mariadb/role/tasks/deploy-standalone.yml b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/tasks/deploy-standalone.yml similarity index 100% rename from MariaDB/10.5.3/mariadb/role/tasks/deploy-standalone.yml rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/tasks/deploy-standalone.yml diff --git a/MariaDB/10.5.3/mariadb/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/tasks/main.yml similarity index 100% rename from MariaDB/10.5.3/mariadb/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/tasks/main.yml diff --git a/MariaDB/10.5.3/mariadb/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/tasks/status-check.yml similarity index 100% rename from MariaDB/10.5.3/mariadb/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/tasks/status-check.yml diff --git a/MariaDB/10.5.3/mariadb/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/tasks/uninstall.yml similarity index 100% rename from MariaDB/10.5.3/mariadb/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/tasks/uninstall.yml diff --git a/MariaDB/10.5.3/mariadb/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/docker-compose.yml.j2 similarity index 100% rename from MariaDB/10.5.3/mariadb/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/docker-compose.yml.j2 diff --git a/MariaDB/10.5.3/mariadb/role/templates/exporter_docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/exporter_docker-compose.yml.j2 similarity index 100% rename from MariaDB/10.5.3/mariadb/role/templates/exporter_docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/exporter_docker-compose.yml.j2 diff --git a/MariaDB/10.5.3/mariadb/role/templates/keepalived/check_mariadb.sh.j2 b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/keepalived/check_mariadb.sh.j2 similarity index 100% rename from MariaDB/10.5.3/mariadb/role/templates/keepalived/check_mariadb.sh.j2 rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/keepalived/check_mariadb.sh.j2 diff --git a/MariaDB/10.5.3/mariadb/role/templates/keepalived/keepalived-mariadb.conf.j2 b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/keepalived/keepalived-mariadb.conf.j2 similarity index 100% rename from MariaDB/10.5.3/mariadb/role/templates/keepalived/keepalived-mariadb.conf.j2 rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/keepalived/keepalived-mariadb.conf.j2 diff --git a/MariaDB/10.5.3/mariadb/role/templates/keepalived/unload_balancer.sh.j2 b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/keepalived/unload_balancer.sh.j2 similarity index 100% rename from MariaDB/10.5.3/mariadb/role/templates/keepalived/unload_balancer.sh.j2 rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/keepalived/unload_balancer.sh.j2 diff --git a/MariaDB/10.5.3/mariadb/role/templates/my.cnf.j2 b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/my.cnf.j2 similarity index 100% rename from MariaDB/10.5.3/mariadb/role/templates/my.cnf.j2 rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/templates/my.cnf.j2 diff --git a/MariaDB/10.5.3/mariadb/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/vars/main.yml similarity index 100% rename from MariaDB/10.5.3/mariadb/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/MariaDB/10.5.3/mariadb/role/vars/main.yml diff --git a/Nacos/2.0.2/hosts b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/hosts similarity index 100% rename from Nacos/2.0.2/hosts rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/hosts diff --git a/Nacos/2.0.2/install.yml b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/install.yml similarity index 100% rename from Nacos/2.0.2/install.yml rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/install.yml diff --git a/Nacos/2.0.2/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/defaults/main.yml similarity index 100% rename from Nacos/2.0.2/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/defaults/main.yml diff --git a/Nacos/2.0.2/role/files/mysql b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/files/mysql similarity index 100% rename from Nacos/2.0.2/role/files/mysql rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/files/mysql diff --git a/Nacos/2.0.2/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/handlers/main.yml similarity index 100% rename from Nacos/2.0.2/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/handlers/main.yml diff --git a/Nacos/2.0.2/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/tasks/deploy.yml similarity index 100% rename from Nacos/2.0.2/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/tasks/deploy.yml diff --git a/Nacos/2.0.2/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/tasks/main.yml similarity index 100% rename from Nacos/2.0.2/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/tasks/main.yml diff --git a/Nacos/2.0.2/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/tasks/status-check.yml similarity index 100% rename from Nacos/2.0.2/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/tasks/status-check.yml diff --git a/Nacos/2.0.2/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/tasks/uninstall.yml similarity index 100% rename from Nacos/2.0.2/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/tasks/uninstall.yml diff --git a/Nacos/2.0.2/role/templates/application.properties.j2 b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/templates/application.properties.j2 similarity index 100% rename from Nacos/2.0.2/role/templates/application.properties.j2 rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/templates/application.properties.j2 diff --git a/Nacos/2.0.2/role/templates/cluster.conf.j2 b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/templates/cluster.conf.j2 similarity index 100% rename from Nacos/2.0.2/role/templates/cluster.conf.j2 rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/templates/cluster.conf.j2 diff --git a/Nacos/2.0.2/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/templates/docker-compose.yml.j2 similarity index 100% rename from Nacos/2.0.2/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/templates/docker-compose.yml.j2 diff --git a/Nacos/2.0.2/role/templates/nacos-logback.xml.j2 b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/templates/nacos-logback.xml.j2 similarity index 100% rename from Nacos/2.0.2/role/templates/nacos-logback.xml.j2 rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/templates/nacos-logback.xml.j2 diff --git a/Nacos/2.0.2/role/templates/nacos-mysql.sql.j2 b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/templates/nacos-mysql.sql.j2 similarity index 100% rename from Nacos/2.0.2/role/templates/nacos-mysql.sql.j2 rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/templates/nacos-mysql.sql.j2 diff --git a/Nacos/2.0.2/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/vars/main.yml similarity index 100% rename from Nacos/2.0.2/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Nacos/2.0.2/role/vars/main.yml diff --git a/Pushgateway/1.4.2/pushgateway/hosts b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/hosts similarity index 100% rename from Pushgateway/1.4.2/pushgateway/hosts rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/hosts diff --git a/Pushgateway/1.4.2/pushgateway/install.yml b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/install.yml similarity index 100% rename from Pushgateway/1.4.2/pushgateway/install.yml rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/install.yml diff --git a/Pushgateway/1.4.2/pushgateway/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/defaults/main.yml similarity index 100% rename from Pushgateway/1.4.2/pushgateway/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/defaults/main.yml diff --git a/Pushgateway/1.4.2/pushgateway/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/handlers/main.yml similarity index 100% rename from Pushgateway/1.4.2/pushgateway/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/handlers/main.yml diff --git a/Pushgateway/1.4.2/pushgateway/role/tasks/deploy.yml b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/tasks/deploy.yml similarity index 100% rename from Pushgateway/1.4.2/pushgateway/role/tasks/deploy.yml rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/tasks/deploy.yml diff --git a/Pushgateway/1.4.2/pushgateway/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/tasks/main.yml similarity index 100% rename from Pushgateway/1.4.2/pushgateway/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/tasks/main.yml diff --git a/Pushgateway/1.4.2/pushgateway/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/tasks/status-check.yml similarity index 100% rename from Pushgateway/1.4.2/pushgateway/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/tasks/status-check.yml diff --git a/Pushgateway/1.4.2/pushgateway/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/tasks/uninstall.yml similarity index 100% rename from Pushgateway/1.4.2/pushgateway/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/tasks/uninstall.yml diff --git a/Pushgateway/1.4.2/pushgateway/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/templates/docker-compose.yml.j2 similarity index 100% rename from Pushgateway/1.4.2/pushgateway/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/templates/docker-compose.yml.j2 diff --git a/Pushgateway/1.4.2/pushgateway/role/vars/.main.yml.swp b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/vars/.main.yml.swp similarity index 100% rename from Pushgateway/1.4.2/pushgateway/role/vars/.main.yml.swp rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/vars/.main.yml.swp diff --git a/Pushgateway/1.4.2/pushgateway/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/vars/main.yml similarity index 100% rename from Pushgateway/1.4.2/pushgateway/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Pushgateway/1.4.2/pushgateway/role/vars/main.yml diff --git a/Redis/6.2.5/redis/hosts b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/hosts similarity index 100% rename from Redis/6.2.5/redis/hosts rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/hosts diff --git a/Redis/6.2.5/redis/install.yml b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/install.yml similarity index 100% rename from Redis/6.2.5/redis/install.yml rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/install.yml diff --git a/Redis/6.2.5/redis/role/defaults/main.yml b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/defaults/main.yml similarity index 100% rename from Redis/6.2.5/redis/role/defaults/main.yml rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/defaults/main.yml diff --git a/Redis/6.2.5/redis/role/handlers/main.yml b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/handlers/main.yml similarity index 100% rename from Redis/6.2.5/redis/role/handlers/main.yml rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/handlers/main.yml diff --git a/Redis/6.2.5/redis/role/tasks/deploy-cluster.yml b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/tasks/deploy-cluster.yml similarity index 100% rename from Redis/6.2.5/redis/role/tasks/deploy-cluster.yml rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/tasks/deploy-cluster.yml diff --git a/Redis/6.2.5/redis/role/tasks/deploy-standalone.yml b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/tasks/deploy-standalone.yml similarity index 100% rename from Redis/6.2.5/redis/role/tasks/deploy-standalone.yml rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/tasks/deploy-standalone.yml diff --git a/Redis/6.2.5/redis/role/tasks/main.yml b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/tasks/main.yml similarity index 100% rename from Redis/6.2.5/redis/role/tasks/main.yml rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/tasks/main.yml diff --git a/Redis/6.2.5/redis/role/tasks/status-check.yml b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/tasks/status-check.yml similarity index 100% rename from Redis/6.2.5/redis/role/tasks/status-check.yml rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/tasks/status-check.yml diff --git a/Redis/6.2.5/redis/role/tasks/uninstall.yml b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/tasks/uninstall.yml similarity index 100% rename from Redis/6.2.5/redis/role/tasks/uninstall.yml rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/tasks/uninstall.yml diff --git a/Redis/6.2.5/redis/role/templates/docker-compose.yml.j2 b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/templates/docker-compose.yml.j2 similarity index 100% rename from Redis/6.2.5/redis/role/templates/docker-compose.yml.j2 rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/templates/docker-compose.yml.j2 diff --git a/Redis/6.2.5/redis/role/templates/redis-master.conf.j2 b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/templates/redis-master.conf.j2 similarity index 100% rename from Redis/6.2.5/redis/role/templates/redis-master.conf.j2 rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/templates/redis-master.conf.j2 diff --git a/Redis/6.2.5/redis/role/templates/redis-slave.conf.j2 b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/templates/redis-slave.conf.j2 similarity index 100% rename from Redis/6.2.5/redis/role/templates/redis-slave.conf.j2 rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/templates/redis-slave.conf.j2 diff --git a/Redis/6.2.5/redis/role/vars/main.yml b/galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/vars/main.yml similarity index 100% rename from Redis/6.2.5/redis/role/vars/main.yml rename to galaxy-data-platform/parcels/roles/components/Redis/6.2.5/redis/role/vars/main.yml diff --git a/tsg-olap/configuration/config.yml b/tsg-olap/configuration/config.yml new file mode 100644 index 0000000..1cd7a1d --- /dev/null +++ b/tsg-olap/configuration/config.yml @@ -0,0 +1,39 @@ +#============================Galaxy OLAP Configuration===============================# +#The default installation location +deploy_dir: /data/hdd/olap + +#The default data storage location,use storing application data,logs and configuration files +data_dir: /data/ssd/olap + +#Use commas (,) to separate the network segments that the firewall allows to access +allowed_ips: 192.168.44.0/24,192.168.45.0/24,192.168.54.0/24,192.168.58.0/24,192.168.41.0/24 + +#============================Keepalived Configuration===============================# +#Specify the interface virtual IP address.It used to configure high availability for OLAP query engine +#Only one IP address can be configured on the same network segment +#The following three ip addresses must be unique +vrrp_instance: + default: + virtual_ipaddress: 192.168.45.102 + interface: eth0 + virtual_router_id: 61 + oss: + virtual_ipaddress: 192.168.45.102 + interface: eth0 + virtual_router_id: 62 + +#============================OSS Configuration===============================# +#HOS token +hos_token: fea0ee76be8147bd9b53ad995b7ef603 + +#HOS token（加密） +encrypted_hos_token: M+0IMYS1+XENltUe585ahhqwY+QZJmnvtxML1vXExB9aO+CPT3GfsP4rtbVuWXpf + +#===========================Central Management Settings==============================# +#Central Management Server IP, Used for Dos detection task get policy. +cm_api: 192.168.44.3 + +#Central Management Server Token, Each environment Token is different. +#Requires communication with CM developers. +cm_api_token: aa2bdec5518ad131f71944b13ce5c298&1& + diff --git a/tsg-olap/configuration/hosts b/tsg-olap/configuration/hosts new file mode 100644 index 0000000..3b820ac --- /dev/null +++ b/tsg-olap/configuration/hosts @@ -0,0 +1,105 @@ +#============================================================================== +# Basic Components +# +# Orchestration & Coordinator & Configuration & Cold Storage +#============================================================================== + +#The cluster use master-master replication mode,maximum 2 servers. +[mariadb] +192.168.45.102 + +#Apache Zookeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services. +#The cluster mode at least 3 servers,The number of nodes must be odd,Like 3/5 nodes. +[zookeeper] +192.168.45.102 + +#Alibaba Nacos an easy-to-use dynamic service discovery, configuration and service management platform +#The cluster mode at least 3 servers,Multi-node HA mode. +[nacos] +192.168.45.102 + +#Apache Hadoop HDFS(Hadoop Distributed File System) +#HDFS is deployed only in cluster mode. +#At least 3 servers,An HDFS cluster consists of two Namenodes and a certain number of Datanodes node. +[hdfs] +192.168.45.102 + +#============================================================================== +# BigData Processing Components +# +# Big data is a term that refers to the massive volume, variety, and velocity of data that is generated from various sources and needs to be stored, processed, and analyzed efficiently. +# The Big Data processing component is used to provide a platform for fast and efficient processing +#============================================================================== + +#Apache Kafka is a distributed event streaming platform,used for high-performance data pipelines, streaming analytics. +#The cluster mode at least 3 servers,By default install CMAK(Management tool) on the first server. +[kafka] +192.168.45.102 + +#Apache Hadoop Yarn,Includes the Flink/Groot-stream runtime environment. +#Yarn is the resource management and job scheduling technology in the open source Hadoop distributed processing framework. +#The cluster mode at least 3 servers,A Yarn cluster consists of two ResourceManager (RM) and a certain number of NodeManager(NM) node. +[yarn] +192.168.45.102 + +#============================================================================== +# Analytic Storage Components +# +# This is a data storage solution designed to support large-scale data analysis and data mining workloads. +# The analytic Storage component it offers high performance, scalability, and flexibility to meet the demands of processing vast amounts of structured and unstructured data. +#============================================================================== + +#Apache HBase is the hosting of very large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware +#The cluster mode at least 3 servers,A HBase cluster consists of three HMaster and a certain number of HRegionServer node. +[hbase] +192.168.45.102 +#Apache Druid is a high performance, real-time analytics database that delivers sub-second queries on streaming and batch data at scale and under load. +#The cluster mode at least 3 servers,A Druid cluster consists of two master/query and a certain number of worker node. +[druid] +192.168.45.102 + +#Yandex ClickHouse is the fastest and most resource efficient open-source database for real-time apps and analytics. +#The cluster mode at least 3 servers,A Clickhouse cluster consists of two query and a certain number of data node. +[clickhouse] +192.168.45.102 + +#ArangoDB is a scalable graph database system to drive value from connected data, faster. +#Only support single server deployment. +[arangodb] +192.168.45.102 + +#============================================================================== +# OLAP Self-research service +# +#============================================================================== + +#The default proxy,Includes the Nginx/Keepalived,If it is a standalone mode, only Nginx. +#A maximum of two nodes. +[loadbalancer] +192.168.45.102 + +#The clickhouse query proxy,Usually deployed with loadbalancer. +[chproxy] +192.168.45.102 + +#Galaxy-hos-service is a distributed object storage service. +#Include components:Keepalived/Nginx/Galaxy-hos-service,If it is a standalone mode, only Galaxy-hos-service/Nginx. +#The cluster mode at least 2 servers,keepalived and nginx services are deployed on the first two nodes by default. +[galaxy_hos_service] +192.168.45.102 + + +#The query gateway,Provides a unified query entry +[galaxy_qgw_service] +192.168.45.102 + +#A lightweight distributed task scheduling framework. +#Include components: Galaxy-job-admin/Galaxy-job-executor +[galaxy_job_service] +192.168.45.102 + +#The report execution service. +[saved_query_scheduler] +192.168.45.102 + + diff --git a/tsg-olap/configuration/tsg-olap.yml b/tsg-olap/configuration/tsg-olap.yml new file mode 100644 index 0000000..9ef347d --- /dev/null +++ b/tsg-olap/configuration/tsg-olap.yml @@ -0,0 +1,93 @@ +config_namespace: "prod" + +# Name of the data center +data_center_name: xxg + +galaxy_qgw_service: + # Running memory of the Galaxy-qgw-service. + java_opts: "-Xms1024m -Xmx3120m -XX:+ExitOnOutOfMemoryError" + +galaxy_job_service: + # Running memory of the Galaxy-job-admin. + admin_java_opts: '-Xms512m -Xmx740m' + # Running memory of the Galaxy-job-executor. + executor_java_opts: '-Xms512m -Xmx1024m' + +saved_query_scheduler: + # Running memory of the saved-query-scheduler. + java_opts: "-Xms512m -Xmx1024m" + +druid: + # Druid job parallelism + index_kafka_statistics_rule: + taskCount: 1 + druid.indexer.runner.javaOpts: "-server -Xms1g -Xmx2g -XX:MaxDirectMemorySize=1g" + +flink: + agg_app_protocol_traffic: + parallelism: 1 + taskmanager.memory.process.size: 2048m + taskmanager.memory.jvm-metaspace.size: 256m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + detection_dos_attack: + parallelism: 1 + taskmanager.memory.process.size: 2048m + taskmanager.memory.jvm-metaspace.size: 256m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + agg_traffic_file_chunk_combine: + parallelism: 1 + taskmanager.memory.process.size: 2048m + taskmanager.memory.jvm-metaspace.size: 256m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + agg_troubleshooting_file_chunk_combine: + parallelism: 1 + taskmanager.memory.process.size: 2048m + taskmanager.memory.jvm-metaspace.size: 256m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + correlation_sip_rtp_session: + parallelism: 1 + taskmanager.memory.process.size: 2048m + taskmanager.memory.jvm-metaspace.size: 256m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + agg_session_record_topn: + parallelism: 1 + taskmanager.memory.process.size: 2048m + taskmanager.memory.jvm-metaspace.size: 256m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + dos_event_kafka_to_clickhouse: + parallelism: 1 + taskmanager.memory.process.size: 2048m + taskmanager.memory.jvm-metaspace.size: 128m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + etl_proxy_event_kafka_to_clickhouse: + parallelism: 1 + taskmanager.memory.process.size: 2048m + taskmanager.memory.jvm-metaspace.size: 256m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + etl_session_record_kafka_to_clickhouse: + parallelism: 1 + taskmanager.memory.process.size: 3072m + taskmanager.memory.jvm-metaspace.size: 256m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + etl_transaction_record_kafka_to_clickhouse: + parallelism: 1 + taskmanager.memory.process.size: 3072m + taskmanager.memory.jvm-metaspace.size: 256m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + etl_voip_record_kafka_to_clickhouse: + parallelism: 1 + taskmanager.memory.process.size: 2048m + taskmanager.memory.jvm-metaspace.size: 128m + taskmanager.numberOfTaskSlots: 1 + taskmanager.memory.framework.off-heap.size: 256m + diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml new file mode 100644 index 0000000..dfb0b32 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml @@ -0,0 +1,11 @@ +- name: Set arangodb check variable + set_fact: + pin: "root:{{ arangodb_default_pin }}" + +- name: Check if collection exists in ArangoDB + uri: + url: "http://{{ groups.arangodb[0] }}:8529/_db/{{ arangodb_database }}/_api/collection/{{ item }}" + method: GET + headers: + Authorization: Basic {{ pin | b64encode }} + loop: "{{ arangodb_collections }}" diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml new file mode 100644 index 0000000..920b6d4 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml @@ -0,0 +1,21 @@ +- name: Creating directory + file: + state: directory + path: '{{ deploy_dir }}/{{ container_name }}/init' + +- name: Copying init files + template: + src: "create_database.sh.j2" + dest: "{{ deploy_dir }}/{{ container_name }}/init/create_database.sh" + mode: 0755 + force: true + +- name: Creating database and collection + shell: 'cd {{ deploy_dir }}/{{ container_name }}/init && sh create_database.sh' + register: response + until: response.rc == 0 + retries: 3 + delay: 3 + failed_when: response.rc != 0 + run_once: true + diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml new file mode 100644 index 0000000..8894d83 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml @@ -0,0 +1,3 @@ +- include: init.yml + when: operation == 'install' +- include: check.yml diff --git a/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 b/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 new file mode 100644 index 0000000..29657da --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 @@ -0,0 +1,54 @@ +#!/bin/bash +source /etc/profile + +arangodb_host={{ groups.arangodb[0] | default('localhost') }} +username=root +pin="{{ arangodb_default_pin }}" +database="{{ arangodb_database }}" + +function create_database() { + # 检查数据库是否存在 + sleep 2 + check_database=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" http://$arangodb_host:8529/_db/$database/_api/database/current) + if [[ $check_database -eq 200 ]]; then + echo "Database $database already exists." + else + # 创建数据库 + local response_code=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" -X POST -H "accept: application/json" -d "{\"name\": \"$database\"}" http://$arangodb_host:8529/_api/database) + if [[ $response_code -eq 201 ]]; then + echo "Database $database created successfully." + else + echo "Failed to create database $database." + exit 1 + fi + fi + sleep 2 + +} + +function create_collection() { + local collection_name=$1 + local collection_type=$2 + sleep 2 + # 检查文档是否存在 + check_collection=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" http://$arangodb_host:8529/_db/$database/_api/collection/$collection_name) + if [[ $check_collection -eq 200 ]]; then + echo "Collection $collection_name already exists." + else + # 创建文档$collection_name + local response_code=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" -X POST -H 'accept: application/json' -d "{\"name\": \"$collection_name\",\"type\": $collection_type }" http://$arangodb_host:8529/_db/$database/_api/collection?isSystem=false) + if [[ $response_code -eq 201 ]]; then + echo "Collection $collection_name created successfully." + else + echo "Failed to create collection $collection_name." + exit 1 + fi + fi + sleep 2 +} + +# 2: document 3: edges +create_database +create_collection IP 2 +create_collection R_LOCATE_FQDN2IP 3 + diff --git a/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml b/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml new file mode 100644 index 0000000..fa8b9d4 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml @@ -0,0 +1,9 @@ +container_name: arangodb + +# 数据库 +arangodb_database: 'tsg_galaxy_v3' + +# 库表 +arangodb_collections: + - IP + - R_LOCATE_FQDN2IP diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql new file mode 100644 index 0000000..7eb3923 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql @@ -0,0 +1,4174 @@ +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster; +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_query; + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,destination_ip,recv_time,log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_query ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + session_rate Int64, + packet_rate Int64, + bit_rate Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,recv_time,log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_query ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); + + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,session_id,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + +alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; +alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,proxy_action,decoded_as,data_center, device_group,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_query ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_asn Nullable(Int64), +server_fqdn String, +server_domain String, +app_transition String,  +app LowCardinality(String), +app_debug_info String, +app_content String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +tunnels String, +dup_traffic_flag Nullable(Int32), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster +TO tsg_galaxy_v3.security_event_local +( + recv_time Int64, + log_id UInt64, + decoded_as String, + session_id UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int32, + tcp_handshake_latency_ms Nullable(Int32), + ingestion_time Int64, + processing_time Int64, + -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), + device_id String, + out_link_id Nullable(Int32), + in_link_id Nullable(Int32), + device_tag String, + data_center String, + device_group String, + sled_ip String, + address_type Int32, + vsys_id Int32, + t_vsys_id Int32, + flags Int64, + flags_identify_info String, + security_rule_list Array(Int64), + security_action String, + monitor_rule_list Array(Int64), + shaping_rule_list Array(Int64), + proxy_rule_list Array(Int64), + statistics_rule_list Array(Int64), + sc_rule_list Array(Int64), + sc_rsp_raw Array(Int64), + sc_rsp_decrypted Array(Int64), + proxy_action String, + proxy_pinning_status Nullable(Int32), + proxy_intercept_status Nullable(Int32), + proxy_passthrough_reason String, + proxy_client_side_latency_ms Nullable(Int32), + proxy_server_side_latency_ms Nullable(Int32), + proxy_client_side_version String, + proxy_server_side_version String, + proxy_cert_verify Nullable(Int32), + proxy_intercept_error String, + monitor_mirrored_pkts Nullable(Int32), + monitor_mirrored_bytes Nullable(Int32), + client_ip String, + client_port Int32, + client_os_desc String, + client_geolocation LowCardinality(String), + client_asn Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + server_ip String, + server_port Int32, + server_os_desc String, + server_geolocation LowCardinality(String), + server_asn Nullable(Int64), + server_fqdn String, + server_domain String, + app_transition String, + app LowCardinality(String), + app_debug_info String, + app_content String, + fqdn_category_list Array(Int64), + ip_protocol LowCardinality(String), + decoded_path LowCardinality(String), + dns_message_id Nullable(Int32), + dns_qr Nullable(Int32), + dns_opcode Nullable(Int32), + dns_aa Nullable(Int32), + dns_tc Nullable(Int32), + dns_rd Nullable(Int32), + dns_ra Nullable(Int32), + dns_rcode Nullable(Int32), + dns_qdcount Nullable(Int32), + dns_ancount Nullable(Int32), + dns_nscount Nullable(Int32), + dns_arcount Nullable(Int32), + dns_qname String, + dns_qtype Nullable(Int32), + dns_qclass Nullable(Int32), + dns_cname String, + dns_sub Nullable(Int32), + dns_rr String, + dns_response_latency_ms Nullable(Int32), + http_url String, + http_host String, + http_request_line String, + http_response_line String, + http_request_body String, + http_response_body String, + http_proxy_flag Nullable(Int32), + http_sequence Nullable(Int32), + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length Nullable(Int64), + http_request_content_type String, + http_response_content_length Nullable(Int64), + http_response_content_type String, + http_set_cookie String, + http_version String, + http_status_code Nullable(Int32), + http_response_latency_ms Nullable(Int32), + http_session_duration_ms Nullable(Int32), + http_action_file_size Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_handshake_latency_ms Nullable(Int32), + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + ssl_esni_flag Nullable(Int32), + ssl_ech_flag Nullable(Int32), + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_handshake_latency_ms Nullable(Int32), + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + mail_protocol_type String, + mail_account String, + mail_from_cmd String, + mail_to_cmd String, + mail_from String, + mail_password String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_eml_file String, + ftp_account String, + ftp_url String, + ftp_link_type String, + quic_version String, + quic_sni String, + quic_user_agent String, + rdp_cookie String, + rdp_security_protocol String, + rdp_client_channels String, + rdp_keyboard_layout String, + rdp_client_version String, + rdp_client_name String, + rdp_client_product_id String, + rdp_desktop_width String, + rdp_desktop_height String, + rdp_requested_color_depth String, + rdp_certificate_type String, + rdp_certificate_count Nullable(Int32), + rdp_certificate_permanent Nullable(Int32), + rdp_encryption_level String, + rdp_encryption_method String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Nullable(Int32), + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Nullable(Int32), + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Nullable(Int32), + sip_bye String, + rtp_payload_type_c2s Nullable(Int32), + rtp_payload_type_s2c Nullable(Int32), + rtp_pcap_path String, + rtp_originator_dir Nullable(Int32), + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + sent_pkts Int64, + received_pkts Int64, + sent_bytes Int64, + received_bytes Int64, + tcp_c2s_ip_fragments Nullable(Int64), + tcp_s2c_ip_fragments Nullable(Int64), + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_rtt_ms Nullable(Int32), + tcp_client_isn Nullable(Int64), + tcp_server_isn Nullable(Int64), + packet_capture_file String, + in_src_mac String, + out_src_mac String, + in_dest_mac String, + out_dest_mac String, + tunnels String, + dup_traffic_flag Nullable(Int32), + tunnel_endpoint_a_desc String, + tunnel_endpoint_b_desc String +) +AS +SELECT + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + proxy_rule_list, + statistics_rule_list, + sc_rule_list, + sc_rsp_raw, + sc_rsp_decrypted, + proxy_action, + proxy_pinning_status, + proxy_intercept_status, + proxy_passthrough_reason, + proxy_client_side_latency_ms, + proxy_server_side_latency_ms, + proxy_client_side_version, + proxy_server_side_version, + proxy_cert_verify, + proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + client_os_desc, + client_geolocation, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + apn, + server_ip, + server_port, + server_os_desc, + server_geolocation, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + app_debug_info, + app_content, + fqdn_category_list, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + ssl_esni_flag, + ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + tunnels, + dup_traffic_flag, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +FROM tsg_galaxy_v3.session_record_local +WHERE empty(security_rule_list) = 0 +; + +-- tsg_galaxy_v3.monitor_event_materialized_view +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster +TO tsg_galaxy_v3.monitor_event_local +( + recv_time Int64, + log_id UInt64, + decoded_as String, + session_id UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int32, + tcp_handshake_latency_ms Nullable(Int32), + ingestion_time Int64, + processing_time Int64, + -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), + device_id String, + out_link_id Nullable(Int32), + in_link_id Nullable(Int32), + device_tag String, + data_center String, + device_group String, + sled_ip String, + address_type Int32, + vsys_id Int32, + t_vsys_id Int32, + flags Int64, + flags_identify_info String, + security_rule_list Array(Int64), + security_action String, + monitor_rule_list Array(Int64), + shaping_rule_list Array(Int64), + proxy_rule_list Array(Int64), + statistics_rule_list Array(Int64), + sc_rule_list Array(Int64), + sc_rsp_raw Array(Int64), + sc_rsp_decrypted Array(Int64), + proxy_action String, + proxy_pinning_status Nullable(Int32), + proxy_intercept_status Nullable(Int32), + proxy_passthrough_reason String, + proxy_client_side_latency_ms Nullable(Int32), + proxy_server_side_latency_ms Nullable(Int32), + proxy_client_side_version String, + proxy_server_side_version String, + proxy_cert_verify Nullable(Int32), + proxy_intercept_error String, + monitor_mirrored_pkts Nullable(Int32), + monitor_mirrored_bytes Nullable(Int32), + client_ip String, + client_port Int32, + client_os_desc String, + client_geolocation LowCardinality(String), + client_asn Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + server_ip String, + server_port Int32, + server_os_desc String, + server_geolocation LowCardinality(String), + server_asn Nullable(Int64), + server_fqdn String, + server_domain String, + app_transition String, + app LowCardinality(String), + app_debug_info String, + app_content String, + fqdn_category_list Array(Int64), + ip_protocol LowCardinality(String), + decoded_path LowCardinality(String), + dns_message_id Nullable(Int32), + dns_qr Nullable(Int32), + dns_opcode Nullable(Int32), + dns_aa Nullable(Int32), + dns_tc Nullable(Int32), + dns_rd Nullable(Int32), + dns_ra Nullable(Int32), + dns_rcode Nullable(Int32), + dns_qdcount Nullable(Int32), + dns_ancount Nullable(Int32), + dns_nscount Nullable(Int32), + dns_arcount Nullable(Int32), + dns_qname String, + dns_qtype Nullable(Int32), + dns_qclass Nullable(Int32), + dns_cname String, + dns_sub Nullable(Int32), + dns_rr String, + dns_response_latency_ms Nullable(Int32), + http_url String, + http_host String, + http_request_line String, + http_response_line String, + http_request_body String, + http_response_body String, + http_proxy_flag Nullable(Int32), + http_sequence Nullable(Int32), + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length Nullable(Int64), + http_request_content_type String, + http_response_content_length Nullable(Int64), + http_response_content_type String, + http_set_cookie String, + http_version String, + http_status_code Nullable(Int32), + http_response_latency_ms Nullable(Int32), + http_session_duration_ms Nullable(Int32), + http_action_file_size Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_handshake_latency_ms Nullable(Int32), + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + ssl_esni_flag Nullable(Int32), + ssl_ech_flag Nullable(Int32), + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_handshake_latency_ms Nullable(Int32), + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + mail_protocol_type String, + mail_account String, + mail_from_cmd String, + mail_to_cmd String, + mail_from String, + mail_password String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_eml_file String, + ftp_account String, + ftp_url String, + ftp_link_type String, + quic_version String, + quic_sni String, + quic_user_agent String, + rdp_cookie String, + rdp_security_protocol String, + rdp_client_channels String, + rdp_keyboard_layout String, + rdp_client_version String, + rdp_client_name String, + rdp_client_product_id String, + rdp_desktop_width String, + rdp_desktop_height String, + rdp_requested_color_depth String, + rdp_certificate_type String, + rdp_certificate_count Nullable(Int32), + rdp_certificate_permanent Nullable(Int32), + rdp_encryption_level String, + rdp_encryption_method String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Nullable(Int32), + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Nullable(Int32), + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Nullable(Int32), + sip_bye String, + rtp_payload_type_c2s Nullable(Int32), + rtp_payload_type_s2c Nullable(Int32), + rtp_pcap_path String, + rtp_originator_dir Nullable(Int32), + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + sent_pkts Int64, + received_pkts Int64, + sent_bytes Int64, + received_bytes Int64, + tcp_c2s_ip_fragments Nullable(Int64), + tcp_s2c_ip_fragments Nullable(Int64), + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_rtt_ms Nullable(Int32), + tcp_client_isn Nullable(Int64), + tcp_server_isn Nullable(Int64), + packet_capture_file String, + in_src_mac String, + out_src_mac String, + in_dest_mac String, + out_dest_mac String, + tunnels String, + dup_traffic_flag Nullable(Int32), + tunnel_endpoint_a_desc String, + tunnel_endpoint_b_desc String +) +AS +SELECT + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + proxy_rule_list, + statistics_rule_list, + sc_rule_list, + sc_rsp_raw, + sc_rsp_decrypted, + proxy_action, + proxy_pinning_status, + proxy_intercept_status, + proxy_passthrough_reason, + proxy_client_side_latency_ms, + proxy_server_side_latency_ms, + proxy_client_side_version, + proxy_server_side_version, + proxy_cert_verify, + proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_port, + client_os_desc, + client_geolocation, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + apn, + server_ip, + server_port, + server_os_desc, + server_geolocation, + server_asn, + server_fqdn, + server_domain, + app_transition, + app, + app_debug_info, + app_content, + fqdn_category_list, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + ssl_esni_flag, + ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + tunnels, + dup_traffic_flag, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +FROM tsg_galaxy_v3.session_record_local +WHERE empty(monitor_rule_list) = 0 +; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql new file mode 100644 index 0000000..1f45639 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql @@ -0,0 +1,9 @@ +create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); +create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); +create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); +create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); +CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); +CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand()); + +alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY; +alter table system.query_log on cluster ck_query modify TTL event_date + INTERVAL 60 DAY; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql new file mode 100644 index 0000000..5e7ba07 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql @@ -0,0 +1,247 @@ +set distributed_ddl_task_timeout = 180; + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_app_full_path String after common_app_id; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_server_asn; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn String after common_server_asn; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64 DEFAULT 1, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_server_domain,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + common_server_domain String, + http_domain String, + ssl_sni String, + common_schema_type LowCardinality(String), + common_vsys_id Int64, + common_client_port Int64, + common_server_port Int64, + common_app_label LowCardinality(String), + common_direction Nullable(Int64), + common_data_center String, + common_device_group String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand()); + + +ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction; + +ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction; + +ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction; + + +ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction; + +ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction; + +ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction; + + +ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; + +ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; + +ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; +ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id; + + +drop view IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster; +drop view IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster; +drop view IF EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster; + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local; + +drop table IF EXISTS `system`.query_log_cluster on cluster ck_query; +drop table IF EXISTS `system`.disks_cluster on cluster ck_query; +drop table IF EXISTS `system`.columns_cluster on cluster ck_query; +drop table IF EXISTS `system`.parts_cluster on cluster ck_query; +drop table IF EXISTS `system`.processes_cluster on cluster ck_query; +drop table IF EXISTS `system`.tables_cluster on cluster ck_query; + + +create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); +create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); +create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); +create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); +CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); +CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand()); +alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY; +alter table system.query_log on cluster ck_query modify TTL event_date + INTERVAL 60 DAY; + diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql new file mode 100644 index 0000000..042b51f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql @@ -0,0 +1,110 @@ +set distributed_ddl_task_timeout = 180; + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS http_request_body_key String after common_flags; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS http_request_body_key String after common_flags; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS http_request_body_key String after common_flags; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info String after common_flags; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info String after common_flags; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids Array(Int64) after common_user_region; + + + + + + + + + + + + + + + + + diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql new file mode 100644 index 0000000..1d3d958 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql @@ -0,0 +1,47 @@ +set distributed_ddl_task_timeout = 180; + +alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY; + + +alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + + +alter table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + + +alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + +alter table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time; +alter table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; +alter table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time; + + + + + diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql new file mode 100644 index 0000000..0e50ac4 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql @@ -0,0 +1,6 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.dos_event_local on cluster ck_cluster add column IF NOT EXISTS profile_id UInt64 after log_id; +ALTER table tsg_galaxy_v3.dos_event on cluster ck_cluster add column IF NOT EXISTS profile_id UInt64 after log_id; +ALTER table tsg_galaxy_v3.dos_event on cluster ck_query add column IF NOT EXISTS profile_id UInt64 after log_id; \ No newline at end of file diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql new file mode 100644 index 0000000..769c47e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql @@ -0,0 +1,11 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; + + +ALTER TABLE tsg_galaxy_v3.proxy_event_local on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; +ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; +ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_query DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql new file mode 100644 index 0000000..11c06a8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql @@ -0,0 +1,159 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie; + + +ALTER TABLE tsg_galaxy_v3.proxy_event_local on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; +ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_cluster DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; +ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_query DROP COLUMN IF EXISTS rdp_encryption_method, DROP COLUMN IF EXISTS rdp_encryption_level, DROP COLUMN IF EXISTS rdp_certificate_permanent, DROP COLUMN IF EXISTS rdp_certificate_count, DROP COLUMN IF EXISTS rdp_certificate_type, DROP COLUMN IF EXISTS rdp_requested_color_depth, DROP COLUMN IF EXISTS rdp_desktop_height, DROP COLUMN IF EXISTS rdp_desktop_width, DROP COLUMN IF EXISTS rdp_client_product_id, DROP COLUMN IF EXISTS rdp_client_name, DROP COLUMN IF EXISTS rdp_client_version, DROP COLUMN IF EXISTS rdp_keyboard_layout, DROP COLUMN IF EXISTS rdp_client_channels, DROP COLUMN IF EXISTS rdp_security_protocol, DROP COLUMN IF EXISTS rdp_cookie; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac String after common_server_domain; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac String after common_server_domain; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac String after common_phone_number; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac String after common_phone_number; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac String after common_phone_number; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql new file mode 100644 index 0000000..27bf30b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql @@ -0,0 +1,97 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; \ No newline at end of file diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql new file mode 100644 index 0000000..df1ba02 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql @@ -0,0 +1,158 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + +ALTER table tsg_galaxy_v3.active_defence_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id; +ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + +ALTER table tsg_galaxy_v3.active_defence_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id; +ALTER table tsg_galaxy_v3.active_defence_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; \ No newline at end of file diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql new file mode 100644 index 0000000..1528e0c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql @@ -0,0 +1,131 @@ +set distributed_ddl_task_timeout = 180; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject; + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time; + + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms; diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql new file mode 100644 index 0000000..20b7e1e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql @@ -0,0 +1,76 @@ +set distributed_ddl_task_timeout = 180; + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port; + + + +ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + +ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port; +ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port; + diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml new file mode 100644 index 0000000..9c4c8b9 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml @@ -0,0 +1,24 @@ +- name: Setting clickhouse init directory + set_fact: + init_path: '{{ deploy_dir }}/clickhouse/init/' + +- name: Creatting clickhouse init directory + file: + path: '{{ init_path }}' + state: directory + +- name: Copying Clickhouse create tables sql + copy: + src: 'files/{{ item }}' + dest: '{{ init_path }}/{{ item }}' + force: true + with_items: + - create_ck_table.sql + - system.sql + +- name: Creating tables + shell: clickhouse-client -h {{ inventory_hostname }} --port 9001 -m -u default --password {{ clickhouse_default_pin }} --multiquery < {{ init_path }}/{{ item }} + loop: + - create_ck_table.sql + - system.sql + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml new file mode 100644 index 0000000..951be61 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml @@ -0,0 +1,5 @@ +- include: init.yml + when: inventory_hostname in groups['clickhouse'][:2] and operation == "install" + +- include: upgrate.yml + when: inventory_hostname in groups['clickhouse'][:2] and operation == "upgrade" diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml new file mode 100644 index 0000000..48401ee --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml @@ -0,0 +1,25 @@ +- name: Setting clickhouse upgrade directory + set_fact: + upgrade_path: '{{ deploy_dir }}/clickhouse/upgrade/' + +- name: Creatting clickhouse upgrade directory + file: + path: '{{ upgrade_path }}' + state: directory + +- name: Copying clickhouse upgrade tables sql + copy: + src: 'files/upgrade-sql' + dest: '{{ upgrade_path }}/' + force: true + +- name: Copying clickhouse upgrade script + template: + src: 'upgrade-clickhouse-data.sh.j2' + dest: '{{ upgrade_path }}/upgrade-clickhouse-data.sh' + mode: 0075 + force: true + +- name: Upgrade clickhouse data + shell: cd {{ upgrade_path }} && sh upgrade-clickhouse-data.sh + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 b/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 new file mode 100644 index 0000000..bc9fd19 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 @@ -0,0 +1,50 @@ +#!/bin/bash +source /etc/profile +set -e + +# 数据库连接参数 +upgrade_dir=${1:-$(pwd)/upgrade-sql} + +# 指定当前版本和历史版本 +current_version="{{ solution_version }}" +old_version="{{ old_version }}" + +# 数据库连接参数 +db_hostname="127.0.0.1" +db_username="default" +db_password="galaxy2019" + +suffix=".sql" +# 获取更新目录中的所有SQL文件 +sql_files=$(find "${upgrade_dir}" -name "*$suffix") + +# 标记是否开始执行历史版本的标识 +start_execution=false + +# 循环处理每个SQL文件 +for file in ${sql_files}; do + # 从文件名中提取版本号 + filename=$(basename "$file") + version=$(echo "$filename" | grep -oE '[0-9]+(\.[0-9]+)?' | tr -d '.') + current_version="${current_version//./}" + old_version="${old_version//./}" + # 检查是否达到历史版本 + if [[ "${version}" -gt "${old_version}" ]]; then + start_execution=true + fi + + # 检查是否开始执行历史版本之后的SQL文件 + if [[ ${start_execution} = true ]]; then + echo "Executing SQL file: ${file}" + + # 执行SQL文件 + clickhouse-client -h 127.0.0.1 --port 9001 -m -u "${db_username}" --password "${db_password}" --multiquery <"${file}" + echo "Upgrade ${version} successfully" + fi + + # 检查是否达到当前版本 + if [[ "${version}" -ge "${current_version}" ]]; then + break + fi +done + diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/druid_segments-tsg.sql b/tsg-olap/parcels/roles/initialization/druid/files/cluster/druid_segments-tsg.sql new file mode 100644 index 0000000..c62f59c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/cluster/druid_segments-tsg.sql @@ -0,0 +1,39 @@ +delete FROM druid.druid_segments where dataSource='application_protocol_stat' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('application_protocol_stat_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-19T10:54:07.120Z', 'application_protocol_stat', '2023-12-19T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-19T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226170706C69636174696F6E5F70726F746F636F6C5F73746174222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31395431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F6170706C69636174696F6E5F70726F746F636F6C5F737461742F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032332D31322D31395431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C70726F746F636F6C5F737461636B5F69642C6170705F6E616D652C767379735F6964222C20226D657472696373223A202273657373696F6E732C6332735F706B74732C7332635F706B74732C6332735F62797465732C7332635F62797465732C6332735F667261676D656E74732C7332635F667261676D656E74732C6332735F7463705F6C6F73745F62797465732C7332635F7463705F6C6F73745F62797465732C6332735F7463705F6F6F6F726465725F706B74732C7332635F7463705F6F6F6F726465725F706B74732C6332735F7463705F72657472616E736D69747465645F706B74732C7332635F7463705F72657472616E736D69747465645F706B74732C6332735F7463705F72657472616E736D69747465645F62797465732C7332635F7463705F72657472616E736D69747465645F62797465732C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20373739312C20226964656E746966696572223A20226170706C69636174696F6E5F70726F746F636F6C5F737461745F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31395431303A35343A30372E3132305A227D); +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('monitor_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-12T10:54:07.120Z', 'monitor_rule_hits', '2023-12-12T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-12T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226D6F6E69746F725F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31325431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F6D6F6E69746F725F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032332D31322D31325431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C616374696F6E222C20226D657472696373223A20226869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333337312C20226964656E746966696572223A20226D6F6E69746F725F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31325431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='object_statistics' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('object_statistics_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-19T10:54:07.120Z', 'object_statistics', '2023-12-19T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-19T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226F626A6563745F73746174697374696373222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31395431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F6F626A6563745F737461746973746963732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032332D31322D31395431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C74656D706C6174655F69642C63686172745F69642C76657273696F6E2C6F626A6563745F747970652C6F626A6563745F69642C6974656D5F6964222C20226D657472696373223A2022696E5F62797465732C6F75745F62797465732C62797465732C6E65775F696E5F73657373696F6E732C6E65775F6F75745F73657373696F6E732C73657373696F6E73222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20343631372C20226964656E746966696572223A20226F626A6563745F737461746973746963735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31395431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='proxy_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('proxy_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'proxy_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A202270726F78795F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F70726F78795F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C7375625F616374696F6E2C70696E6E696E675F7374617475732C72756C655F69642C616374696F6E2C6869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333736352C20226964656E746966696572223A202270726F78795F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('security_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-12T10:54:07.120Z', 'security_rule_hits', '2023-12-12T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-12T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A202273656375726974795F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31325431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F73656375726974795F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032332D31322D31325431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C616374696F6E222C20226D657472696373223A20226869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333337312C20226964656E746966696572223A202273656375726974795F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31325431303A35343A30372E3132305A227D); + +delete FROM druid.druid_segments where dataSource='service_chaining_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('service_chaining_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'service_chaining_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022736572766963655F636861696E696E675F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F736572766963655F636861696E696E675F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C7366665F70726F66696C655F69642C73665F70726F66696C655F69642C73656E745F706B74732C73656E745F62797465732C726563765F706B74732C726563765F6279746573222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333132322C20226964656E746966696572223A2022736572766963655F636861696E696E675F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='service_function_status' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('service_function_status_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'service_function_status', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022736572766963655F66756E6374696F6E5F737461747573222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F736572766963655F66756E6374696F6E5F7374617475732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C73665F70726F66696C655F69642C73665F7374617475732C73665F6C6174656E63795F7573222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323333312C20226964656E746966696572223A2022736572766963655F66756E6374696F6E5F7374617475735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='statistics_rule' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('statistics_rule_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'statistics_rule', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022737461746973746963735F72756C65222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F737461746973746963735F72756C652F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C74656D706C6174655F69642C63686172745F69642C76657273696F6E2C636C69656E745F69705F6F626A6563742C7365727665725F69705F6F626A6563742C6671646E5F63617465676F72792C636C69656E745F69702C7365727665725F69702C7365727665725F6671646E2C7365727665725F646F6D61696E2C6170706C69636174696F6E2C696E5F62797465732C6F75745F62797465732C62797465732C6E65775F6332735F666C6F77732C6E65775F7332635F666C6F77732C6E65775F696E5F73657373696F6E732C6E65775F6F75745F73657373696F6E732C73657373696F6E732C6E65775F756E65737461626C69736865645F73657373696F6E732C73796E5F706B7473222C20226D657472696373223A202273657373696F6E5F6964656E7469666965725F736B657463682C636C69656E745F69705F736B657463682C7365727665725F69705F736B657463682C696E5F6C6174656E63795F6D735F736B657463682C6F75745F6C6174656E63795F6D735F736B657463682C6C6174656E63795F6D735F736B65746368222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20393433362C20226964656E746966696572223A2022737461746973746963735F72756C655F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='statistics_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('statistics_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'statistics_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022737461746973746963735F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F737461746973746963735F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C6869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323838302C20226964656E746966696572223A2022737461746973746963735F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='sys_storage_log' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('sys_storage_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'sys_storage_log', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20227379735F73746F726167655F6C6F67222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F7379735F73746F726167655F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226C6F675F747970652C646174615F63656E7465722C6167677265676174655F73697A652C6C6173745F73746F726167652C6D61785F73697A652C757365645F73697A65222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20313938322C20226964656E746966696572223A20227379735F73746F726167655F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_client_ips' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_client_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_client_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F636C69656E745F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F636C69656E745F6970732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022636C69656E745F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333335362C20226964656E746966696572223A2022746F705F636C69656E745F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_external_ips' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_external_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_external_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F65787465726E616C5F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F65787465726E616C5F6970732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A202265787465726E616C5F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336322C20226964656E746966696572223A2022746F705F65787465726E616C5F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_internal_ips' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_internal_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_internal_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F696E7465726E616C5F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F696E7465726E616C5F6970732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022696E7465726E616C5F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336322C20226964656E746966696572223A2022746F705F696E7465726E616C5F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_server_domains' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_domains_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_domains', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F646F6D61696E73222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F7365727665725F646F6D61696E732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022646F6D61696E2C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333334372C20226964656E746966696572223A2022746F705F7365727665725F646F6D61696E735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_server_fqdns' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_fqdns_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_fqdns', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F6671646E73222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F7365727665725F6671646E732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226671646E2C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333334312C20226964656E746966696572223A2022746F705F7365727665725F6671646E735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_server_ips' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F7365727665725F6970732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20227365727665725F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333335362C20226964656E746966696572223A2022746F705F7365727665725F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_subscribers' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_subscribers_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_subscribers', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7375627363726962657273222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F73756273637269626572732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022737562736372696265725F69642C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336382C20226964656E746966696572223A2022746F705F73756273637269626572735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='traffic_general_stat' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_general_stat_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-26T10:54:07.120Z', 'traffic_general_stat', '2023-12-26T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-26T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F67656E6572616C5F73746174222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D32365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F67656E6572616C5F737461742F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032332D31322D32365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C767379735F6964222C20226D657472696373223A202273657373696F6E732C6163746976655F73657373696F6E732C636C6F7365645F73657373696F6E732C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C6173796D6D65747269635F6332735F666C6F77732C6173796D6D65747269635F7332635F666C6F77732C7463705F73796E5F706B74732C7463705F61636B5F706B74732C7463705F66696E5F706B74732C7463705F7273745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20353235322C20226964656E746966696572223A2022747261666669635F67656E6572616C5F737461745F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D32365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='traffic_shaping_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_shaping_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'traffic_shaping_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F73686170696E675F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F73686170696E675F72756C655F686974732F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C70726F66696C655F69642C7072696F726974792C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C696E5F64726F705F706B74732C6F75745F64726F705F706B74732C696E5F6D61785F6C6174656E63795F75732C6F75745F6D61785F6C6174656E63795F75732C696E5F71756575655F6C656E2C6F75745F71756575655F6C656E222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20343335332C20226964656E746966696572223A2022747261666669635F73686170696E675F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='traffic_top_destination_ip_metrics_log' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_top_destination_ip_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'traffic_top_destination_ip_metrics_log', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F67222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A202268646673222C202270617468223A2022686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032342D30312D31365431305F35345F30372E3132305A2F305F696E6465782E7A6970227D2C202264696D656E73696F6E73223A202261747461636B5F747970652C64657374696E6174696F6E5F69702C6269745F726174652C7061636B65745F726174652C706172746974696F6E5F6E756D2C73657373696F6E5F726174652C767379735F6964222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323139332C20226964656E746966696572223A2022747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/push_segments.sh b/tsg-olap/parcels/roles/initialization/druid/files/cluster/push_segments.sh new file mode 100644 index 0000000..3a5410e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/cluster/push_segments.sh @@ -0,0 +1,28 @@ +#!/bin/bash +source /etc/profile + +TIME_DIR="30000101T000000.000Z_30000102T000000.000Z" + +hadoop fs -test -e /druid/segments +if [ $? -ne 0 ] ;then +hadoop fs -mkdir -p /druid/segments +fi + +for dir in `ls ./segments` +do + hadoop fs -test -e /druid/segments/$dir/$TIME_DIR + if [ $? -eq 0 ] ;then + for file in `ls ./segments/$dir/$TIME_DIR` + do + hadoop fs -test -e /druid/segments/$dir/$TIME_DIR/$file + if [ $? -ne 0 ] ;then + hadoop fs -put ./segments/$dir/$TIME_DIR/$file /druid/segments/$dir/$TIME_DIR/ + fi + done + else + hadoop fs -put ./segments/$dir /druid/segments/ + fi + +done + + diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_data_rule.json b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_data_rule.json new file mode 100644 index 0000000..ad1c808 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_data_rule.json @@ -0,0 +1,13 @@ +[ + { + "type" : "loadByPeriod", + "period" : "P365D", + "includeFuture" : true, + "tieredReplicants": { + "_default_tier" : 1 + } + }, + { + "type" : "dropForever" + } +] diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_task.txt b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_task.txt new file mode 100644 index 0000000..f3e8373 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/common_task.txt @@ -0,0 +1,19 @@ +application_protocol_stat +proxy_rule_hits +security_rule_hits +service_chaining_rule_hits +service_function_status +statistics_object +statistics_rule_hits +statistics_rule +sys_storage_log +top_client_ips +top_external_ips +top_internal_ips +top_server_domains +top_server_fqdns +top_server_ips +top_subscribers +traffic_general_stat +traffic_shaping_rule_hits +traffic_top_destination_ip_metrics_log diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/hot_data_rule.json b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/hot_data_rule.json new file mode 100644 index 0000000..a44486e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/hot_data_rule.json @@ -0,0 +1,13 @@ +[ + { + "type" : "loadByPeriod", + "period" : "PT1H", + "includeFuture" : true, + "tieredReplicants": { + "_default_tier" : 1 + } + }, + { + "type" : "dropForever" + } +] diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/hot_task.txt b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/hot_task.txt new file mode 100644 index 0000000..e69de29 diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/post_rule.sh b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/post_rule.sh new file mode 100644 index 0000000..0889f1d --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/post_rule.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +base_dir=`dirname $0` +echo $base_dir + +#router节点IP +router_ip=$1 + +common_task_name=`cat $base_dir/common_task.txt` +common_rule_file="$base_dir/common_data_rule.json" + +hot_task_name=`cat $base_dir/hot_task.txt` +hot_rule_file="$base_dir/hot_data_rule.json" + +for var in ${common_task_name[@]}; +do +curl -X 'POST' -H 'Content-Type:application/json' -d @$common_rule_file http://${router_ip}:8089/druid/coordinator/v1/rules/${var} +echo "'${var}' 任务启动成功" +sleep 2 +done + +for var in ${hot_task_name[@]}; +do +curl -X 'POST' -H 'Content-Type:application/json' -d @$hot_rule_file http://${router_ip}:8089/druid/coordinator/v1/rules/${var} +echo "'${var}' 任务启动成功" +sleep 2 +done + diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/supervisor-manger b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/supervisor-manger new file mode 100644 index 0000000..799da22 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/cluster/rule/supervisor-manger @@ -0,0 +1,76 @@ +#!/bin/bash + +base_dir=`dirname $0` +base_dir="$(cd "$base_dir" && pwd)" +#echo $base_dir + +router_ip='127.0.0.1' + +common_task_name=`cat $base_dir/common_task.txt` +common_rule_file="$base_dir/common_data_rule.json" + +hot_task_name=`cat $base_dir/hot_task.txt` +hot_rule_file="$base_dir/hot_data_rule.json" + +case $1 in + + resetAll) + for var in ${common_task_name[@]}; + do + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset + sleep 2 + done + + for var in ${hot_task_name[@]}; + do + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset + sleep 2 + done + ;; + + terminateAll) + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/terminateAll + ;; + + shutdownAllTasks) + for var in ${common_task_name[@]}; + do + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks + sleep 2 + done + + for var in ${hot_task_name[@]}; + do + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks + sleep 2 + done + ;; + + reset) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger reset " + exit 1 + fi + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/reset + ;; + + terminate) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger terminate " + exit 1 + fi + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/terminate + ;; + + shutdownTasks) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger shutdownTasks " + exit 1 + fi + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/$2/shutdownAllTasks + ;; + *) + echo "Usage: supervisor-manger {resetAll|terminateAll|shutdownAllTasks}" + echo "Usage: supervisor-manger {reset|terminate|shutdownTasks} " + ;; +esac diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments.zip new file mode 100644 index 0000000..cacbbe1 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/application_protocol_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/application_protocol_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..af00337 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/application_protocol_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/monitor_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/monitor_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..6989ff6 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/monitor_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/object_statistics/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/object_statistics/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..8f6fe84 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/object_statistics/30000101T000000.000Z_30000102T000000.000Z/2023-12-19T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/proxy_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/proxy_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..ac74b6a Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/proxy_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/security_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/security_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..6989ff6 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/security_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2023-12-12T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_chaining_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_chaining_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..e5ae0ef Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_chaining_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_function_status/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_function_status/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..5062936 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/service_function_status/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..2424df4 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..913226a Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/statistics_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/sys_storage_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/sys_storage_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..22590a9 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/sys_storage_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_client_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_client_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..46e2772 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_client_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_external_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_external_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..24cefaa Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_external_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_internal_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_internal_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..79e9fb5 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_internal_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_domains/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_domains/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..ae93e8d Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_domains/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_fqdns/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_fqdns/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..8799887 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_fqdns/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..0926f1f Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_server_ips/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_subscribers/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_subscribers/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..e27ddcd Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/top_subscribers/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_general_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-26T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_general_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-26T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..66c55ae Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_general_stat/30000101T000000.000Z_30000102T000000.000Z/2023-12-26T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_shaping_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_shaping_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..84c033d Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_shaping_rule_hits/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_top_destination_ip_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_top_destination_ip_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip new file mode 100644 index 0000000..6305798 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/cluster/segments/traffic_top_destination_ip_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2024-01-16T10_54_07.120Z/0_index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/mysql b/tsg-olap/parcels/roles/initialization/druid/files/mysql new file mode 100644 index 0000000..eb26146 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/mysql differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/rule/common_data_rule.json b/tsg-olap/parcels/roles/initialization/druid/files/rule/common_data_rule.json new file mode 100644 index 0000000..ad1c808 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/rule/common_data_rule.json @@ -0,0 +1,13 @@ +[ + { + "type" : "loadByPeriod", + "period" : "P365D", + "includeFuture" : true, + "tieredReplicants": { + "_default_tier" : 1 + } + }, + { + "type" : "dropForever" + } +] diff --git a/tsg-olap/parcels/roles/initialization/druid/files/rule/common_task.txt b/tsg-olap/parcels/roles/initialization/druid/files/rule/common_task.txt new file mode 100644 index 0000000..6f6e7f1 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/rule/common_task.txt @@ -0,0 +1,19 @@ +traffic_general_stat.json +application_protocol_stat.json +traffic_shaping_rule_hits.json +traffic_top_destination_ip_metrics_log.json +service_chaining_rule_hits.json +security_rule_hits.json +proxy_rule_hits.json +service_function_status.json +sys_storage_log.json +top_client_ips.json +top_subscribers.json +top_external_ips.json +top_server_ips.json +top_internal_ips.json +top_server_fqdns.json +top_server_domains.json +statistics_rule.json +statistics_rule_hits.json +object_statistics.json diff --git a/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_data_rule.json b/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_data_rule.json new file mode 100644 index 0000000..a44486e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_data_rule.json @@ -0,0 +1,13 @@ +[ + { + "type" : "loadByPeriod", + "period" : "PT1H", + "includeFuture" : true, + "tieredReplicants": { + "_default_tier" : 1 + } + }, + { + "type" : "dropForever" + } +] diff --git a/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_task.txt b/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_task.txt new file mode 100644 index 0000000..9bfa834 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/rule/hot_task.txt @@ -0,0 +1,2 @@ +urls_proxy_hot +urls_security_hot diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/druid_segments-tsg.sql b/tsg-olap/parcels/roles/initialization/druid/files/standalone/druid_segments-tsg.sql new file mode 100644 index 0000000..69118f4 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/files/standalone/druid_segments-tsg.sql @@ -0,0 +1,39 @@ +delete FROM druid.druid_segments where dataSource='application_protocol_stat' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('application_protocol_stat_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-19T10:54:07.120Z', 'application_protocol_stat', '2023-12-19T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-19T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226170706C69636174696F6E5F70726F746F636F6C5F73746174222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31395431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F6170706C69636174696F6E5F70726F746F636F6C5F737461742F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032332D31322D31395431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C70726F746F636F6C5F737461636B5F69642C6170705F6E616D652C767379735F6964222C20226D657472696373223A202273657373696F6E732C6332735F706B74732C7332635F706B74732C6332735F62797465732C7332635F62797465732C6332735F667261676D656E74732C7332635F667261676D656E74732C6332735F7463705F6C6F73745F62797465732C7332635F7463705F6C6F73745F62797465732C6332735F7463705F6F6F6F726465725F706B74732C7332635F7463705F6F6F6F726465725F706B74732C6332735F7463705F72657472616E736D69747465645F706B74732C7332635F7463705F72657472616E736D69747465645F706B74732C6332735F7463705F72657472616E736D69747465645F62797465732C7332635F7463705F72657472616E736D69747465645F62797465732C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20373739312C20226964656E746966696572223A20226170706C69636174696F6E5F70726F746F636F6C5F737461745F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31395431303A35343A30372E3132305A227D); +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('monitor_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-12T10:54:07.120Z', 'monitor_rule_hits', '2023-12-12T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-12T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226D6F6E69746F725F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31325431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F6D6F6E69746F725F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032332D31322D31325431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C616374696F6E222C20226D657472696373223A20226869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333337312C20226964656E746966696572223A20226D6F6E69746F725F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31325431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='object_statistics' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('object_statistics_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-19T10:54:07.120Z', 'object_statistics', '2023-12-19T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-19T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20226F626A6563745F73746174697374696373222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31395431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F6F626A6563745F737461746973746963732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032332D31322D31395431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C74656D706C6174655F69642C63686172745F69642C76657273696F6E2C6F626A6563745F747970652C6F626A6563745F69642C6974656D5F6964222C20226D657472696373223A2022696E5F62797465732C6F75745F62797465732C62797465732C6E65775F696E5F73657373696F6E732C6E65775F6F75745F73657373696F6E732C73657373696F6E73222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20343631372C20226964656E746966696572223A20226F626A6563745F737461746973746963735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31395431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='proxy_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('proxy_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'proxy_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A202270726F78795F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F70726F78795F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C7375625F616374696F6E2C70696E6E696E675F7374617475732C72756C655F69642C616374696F6E2C6869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333736352C20226964656E746966696572223A202270726F78795F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('security_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-12T10:54:07.120Z', 'security_rule_hits', '2023-12-12T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-12T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A202273656375726974795F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D31325431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F73656375726974795F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032332D31322D31325431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C616374696F6E222C20226D657472696373223A20226869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333337312C20226964656E746966696572223A202273656375726974795F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D31325431303A35343A30372E3132305A227D); + +delete FROM druid.druid_segments where dataSource='service_chaining_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('service_chaining_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'service_chaining_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022736572766963655F636861696E696E675F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F736572766963655F636861696E696E675F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C7366665F70726F66696C655F69642C73665F70726F66696C655F69642C73656E745F706B74732C73656E745F62797465732C726563765F706B74732C726563765F6279746573222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333132322C20226964656E746966696572223A2022736572766963655F636861696E696E675F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='service_function_status' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('service_function_status_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'service_function_status', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022736572766963655F66756E6374696F6E5F737461747573222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F736572766963655F66756E6374696F6E5F7374617475732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C73665F70726F66696C655F69642C73665F7374617475732C73665F6C6174656E63795F7573222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323333312C20226964656E746966696572223A2022736572766963655F66756E6374696F6E5F7374617475735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='statistics_rule' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('statistics_rule_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'statistics_rule', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022737461746973746963735F72756C65222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F737461746973746963735F72756C652F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C74656D706C6174655F69642C63686172745F69642C76657273696F6E2C636C69656E745F69705F6F626A6563742C7365727665725F69705F6F626A6563742C6671646E5F63617465676F72792C636C69656E745F69702C7365727665725F69702C7365727665725F6671646E2C7365727665725F646F6D61696E2C6170706C69636174696F6E2C696E5F62797465732C6F75745F62797465732C62797465732C6E65775F6332735F666C6F77732C6E65775F7332635F666C6F77732C6E65775F696E5F73657373696F6E732C6E65775F6F75745F73657373696F6E732C73657373696F6E732C6E65775F756E65737461626C69736865645F73657373696F6E732C73796E5F706B7473222C20226D657472696373223A202273657373696F6E5F6964656E7469666965725F736B657463682C636C69656E745F69705F736B657463682C7365727665725F69705F736B657463682C696E5F6C6174656E63795F6D735F736B657463682C6F75745F6C6174656E63795F6D735F736B657463682C6C6174656E63795F6D735F736B65746368222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20393433362C20226964656E746966696572223A2022737461746973746963735F72756C655F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='statistics_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('statistics_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'statistics_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022737461746973746963735F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F737461746973746963735F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C6869745F636F756E742C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B7473222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323838302C20226964656E746966696572223A2022737461746973746963735F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='sys_storage_log' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('sys_storage_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'sys_storage_log', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A20227379735F73746F726167655F6C6F67222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F7379735F73746F726167655F6C6F672F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226C6F675F747970652C646174615F63656E7465722C6167677265676174655F73697A652C6C6173745F73746F726167652C6D61785F73697A652C757365645F73697A65222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20313938322C20226964656E746966696572223A20227379735F73746F726167655F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_client_ips' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_client_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_client_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F636C69656E745F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F636C69656E745F6970732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022636C69656E745F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333335362C20226964656E746966696572223A2022746F705F636C69656E745F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_external_ips' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_external_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_external_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F65787465726E616C5F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F65787465726E616C5F6970732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A202265787465726E616C5F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336322C20226964656E746966696572223A2022746F705F65787465726E616C5F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_internal_ips' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_internal_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_internal_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F696E7465726E616C5F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F696E7465726E616C5F6970732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022696E7465726E616C5F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336322C20226964656E746966696572223A2022746F705F696E7465726E616C5F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_server_domains' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_domains_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_domains', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F646F6D61696E73222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F7365727665725F646F6D61696E732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022646F6D61696E2C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333334372C20226964656E746966696572223A2022746F705F7365727665725F646F6D61696E735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_server_fqdns' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_fqdns_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_fqdns', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F6671646E73222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F7365727665725F6671646E732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226671646E2C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333334312C20226964656E746966696572223A2022746F705F7365727665725F6671646E735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_server_ips' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_server_ips_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_server_ips', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7365727665725F697073222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F7365727665725F6970732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20227365727665725F69702C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333335362C20226964656E746966696572223A2022746F705F7365727665725F6970735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='top_subscribers' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('top_subscribers_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'top_subscribers', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022746F705F7375627363726962657273222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F746F705F73756273637269626572732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022737562736372696265725F69642C646174615F63656E7465722C6465766963655F67726F75702C6465766963655F69642C767379735F69642C6D65747269632C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C73657373696F6E73222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20333336382C20226964656E746966696572223A2022746F705F73756273637269626572735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='traffic_general_stat' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_general_stat_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2023-12-26T10:54:07.120Z', 'traffic_general_stat', '2023-12-26T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2023-12-26T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F67656E6572616C5F73746174222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032332D31322D32365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F747261666669635F67656E6572616C5F737461742F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032332D31322D32365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A20226465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C767379735F6964222C20226D657472696373223A202273657373696F6E732C6163746976655F73657373696F6E732C636C6F7365645F73657373696F6E732C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C6173796D6D65747269635F6332735F666C6F77732C6173796D6D65747269635F7332635F666C6F77732C7463705F73796E5F706B74732C7463705F61636B5F706B74732C7463705F66696E5F706B74732C7463705F7273745F706B7473222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20353235322C20226964656E746966696572223A2022747261666669635F67656E6572616C5F737461745F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032332D31322D32365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='traffic_shaping_rule_hits' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_shaping_rule_hits_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'traffic_shaping_rule_hits', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F73686170696E675F72756C655F68697473222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F747261666669635F73686170696E675F72756C655F686974732F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A2022767379735F69642C6465766963655F69642C6465766963655F67726F75702C646174615F63656E7465722C72756C655F69642C70726F66696C655F69642C7072696F726974792C696E5F62797465732C6F75745F62797465732C696E5F706B74732C6F75745F706B74732C696E5F64726F705F706B74732C6F75745F64726F705F706B74732C696E5F6D61785F6C6174656E63795F75732C6F75745F6D61785F6C6174656E63795F75732C696E5F71756575655F6C656E2C6F75745F71756575655F6C656E222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20343335332C20226964656E746966696572223A2022747261666669635F73686170696E675F72756C655F686974735F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); +delete FROM druid.druid_segments where dataSource='traffic_top_destination_ip_metrics_log' and `start` = '3000-01-02T00:00:00.000Z'; +REPLACE INTO druid.druid_segments (id, dataSource, created_date, `start`, `end`, partitioned, version, used, payload) VALUES ('traffic_top_destination_ip_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2024-01-16T10:54:07.120Z', 'traffic_top_destination_ip_metrics_log', '2024-01-16T10:54:07.120Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2024-01-16T10:54:07.120Z', 1, 0x7B2264617461536F75726365223A2022747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F67222C2022696E74657276616C223A2022333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C202276657273696F6E223A2022323032342D30312D31365431303A35343A30372E3132305A222C20226C6F616453706563223A207B2274797065223A20226C6F63616C222C202270617468223A20222F6170616368652D64727569642D32362E302E302F7661722F64727569642F7365676D656E74732F747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F672F333030302D30312D30315430303A30303A30302E3030305A5F333030302D30312D30325430303A30303A30302E3030305A2F323032342D30312D31365431303A35343A30372E3132305A2F302F696E6465782E7A6970227D2C202264696D656E73696F6E73223A202261747461636B5F747970652C64657374696E6174696F6E5F69702C6269745F726174652C7061636B65745F726174652C706172746974696F6E5F6E756D2C73657373696F6E5F726174652C767379735F6964222C20226D657472696373223A2022222C2022736861726453706563223A207B2274797065223A20226E756D6265726564222C2022706172746974696F6E4E756D223A20302C2022706172746974696F6E73223A20307D2C202262696E61727956657273696F6E223A20392C202273697A65223A20323139332C20226964656E746966696572223A2022747261666669635F746F705F64657374696E6174696F6E5F69705F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032342D30312D31365431303A35343A30372E3132305A227D); diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments.zip new file mode 100644 index 0000000..7065ed6 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/application_protocol_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-12-19T10_54_07.120Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/application_protocol_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-12-19T10_54_07.120Z/0/index.zip new file mode 100644 index 0000000..af00337 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/application_protocol_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-12-19T10_54_07.120Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/object_statistics/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-24T02_43_00.245Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/object_statistics/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-24T02_43_00.245Z/0/index.zip new file mode 100644 index 0000000..315ce74 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/object_statistics/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-24T02_43_00.245Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/proxy_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-10-18T10_44_50.148Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/proxy_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-10-18T10_44_50.148Z/0/index.zip new file mode 100644 index 0000000..fc74b52 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/proxy_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-10-18T10_44_50.148Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/security_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-21T10_23_31.391Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/security_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-21T10_23_31.391Z/0/index.zip new file mode 100644 index 0000000..db20bfb Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/security_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-21T10_23_31.391Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_chaining_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-17T06_26_28.371Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_chaining_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-17T06_26_28.371Z/0/index.zip new file mode 100644 index 0000000..1218d51 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_chaining_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-17T06_26_28.371Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_function_status/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-02-10T03_07_09.579Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_function_status/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-02-10T03_07_09.579Z/0/index.zip new file mode 100644 index 0000000..4d98e20 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/service_function_status/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-02-10T03_07_09.579Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-07-28T07_05_53.280Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-07-28T07_05_53.280Z/0/index.zip new file mode 100644 index 0000000..7a46b17 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-07-28T07_05_53.280Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-04T01_58_01.118Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-04T01_58_01.118Z/0/index.zip new file mode 100644 index 0000000..e45f8a9 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/statistics_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-08-04T01_58_01.118Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/sys_storage_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-02-04T03_40_23.597Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/sys_storage_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-02-04T03_40_23.597Z/0/index.zip new file mode 100644 index 0000000..7958e35 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/sys_storage_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-02-04T03_40_23.597Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-23T07_38_34.276Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-23T07_38_34.276Z/0/index.zip new file mode 100644 index 0000000..6e66708 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-23T07_38_34.276Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T09_35_24.409Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T09_35_24.409Z/0/index.zip new file mode 100644 index 0000000..0d6cf04 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T09_35_24.409Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-05T02_14_42.249Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-05T02_14_42.249Z/0/index.zip new file mode 100644 index 0000000..a5236eb Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_client_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-05T02_14_42.249Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_external_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_37.283Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_external_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_37.283Z/0/index.zip new file mode 100644 index 0000000..9bd0145 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_external_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_37.283Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_internal_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_30.600Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_internal_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_30.600Z/0/index.zip new file mode 100644 index 0000000..fc955b0 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_internal_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_39_30.600Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_domains/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T06_21_09.765Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_domains/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T06_21_09.765Z/0/index.zip new file mode 100644 index 0000000..5210fee Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_domains/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T06_21_09.765Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_fqdns/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T09_15_58.356Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_fqdns/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T09_15_58.356Z/0/index.zip new file mode 100644 index 0000000..4c9c033 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_fqdns/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-27T09_15_58.356Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T07_40_42.170Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T07_40_42.170Z/0/index.zip new file mode 100644 index 0000000..7fac0da Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_server_ips/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T07_40_42.170Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_subscribers/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_40_14.760Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_subscribers/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_40_14.760Z/0/index.zip new file mode 100644 index 0000000..2b6a611 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/top_subscribers/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-24T06_40_14.760Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-18T02_05_57.690Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-18T02_05_57.690Z/0/index.zip new file mode 100644 index 0000000..90e2d7f Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-04-18T02_05_57.690Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_19_29.661Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_19_29.661Z/0/index.zip new file mode 100644 index 0000000..4e201d2 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_19_29.661Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_43_09.655Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_43_09.655Z/0/index.zip new file mode 100644 index 0000000..e21f94b Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_43_09.655Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_50_55.510Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_50_55.510Z/0/index.zip new file mode 100644 index 0000000..6964bd4 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_50_55.510Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_52_58.683Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_52_58.683Z/0/index.zip new file mode 100644 index 0000000..91ff68a Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_52_58.683Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_53_44.028Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_53_44.028Z/0/index.zip new file mode 100644 index 0000000..aad2f86 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T06_53_44.028Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_06_43.540Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_06_43.540Z/0/index.zip new file mode 100644 index 0000000..b4c4600 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_06_43.540Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_26_53.933Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_26_53.933Z/0/index.zip new file mode 100644 index 0000000..51dd724 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_26_53.933Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_50_48.464Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_50_48.464Z/0/index.zip new file mode 100644 index 0000000..9fbb526 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_general_stat/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-05-06T07_50_48.464Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_shaping_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-03-30T07_05_43.972Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_shaping_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-03-30T07_05_43.972Z/0/index.zip new file mode 100644 index 0000000..df83b3e Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_shaping_rule_hits/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2023-03-30T07_05_43.972Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_10_50.930Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_10_50.930Z/0/index.zip new file mode 100644 index 0000000..48e44f6 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_10_50.930Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_15_36.184Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_15_36.184Z/0/index.zip new file mode 100644 index 0000000..7467dc8 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2021-08-11T10_15_36.184Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2022-09-06T06_57_25.946Z/0/index.zip b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2022-09-06T06_57_25.946Z/0/index.zip new file mode 100644 index 0000000..51dbbf7 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/druid/files/standalone/segments/traffic_top_destination_ip_metrics_log/3000-01-01T00_00_00.000Z_3000-01-02T00_00_00.000Z/2022-09-06T06_57_25.946Z/0/index.zip differ diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml new file mode 100644 index 0000000..899461b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml @@ -0,0 +1,25 @@ +- name: Check if backup directory exists + stat: + path: "{{ backup_path }}" + register: exist_status + +- name: Backup directories + block: + - name: Creat backup directory + file: + state: directory + path: "{{ backup_path }}" + + - name: Copying directories to backup + copy: + src: "{{ deploy_dir }}/{{ container_name }}/{{ item }}" + dest: "{{ backup_path }}" + remote_src: true + loop: "{{ backup_items }}" + + rescue: + - name: Remove backup directory on failure + file: + path: "{{ backup_path }}" + state: absent + when: exist_status.stat.exists == false diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml new file mode 100644 index 0000000..295e895 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml @@ -0,0 +1,23 @@ +- name: Get druid running tasks + shell: curl -s http://{{ vrrp_instance.default.virtual_ipaddress }}:8089/druid/indexer/v1/supervisor?state=true | jq -r '.[] | select(.state == "RUNNING") | .id' + register: run_task_list + run_once: true + +- name: Get init task template + find: + paths: "templates/tasks/" + file_type: file + patterns: "*.json" + register: find_result + delegate_to: 127.0.0.1 + run_once: true + +- name: Set init task list + set_fact: + init_task_list: "{{ find_result.files | map(attribute='path') | map('basename') | map('regex_replace', '^index_kafka_(.*)\\.json$', '\\1') | list }}" + +- name: Verify that the number of running tasks is correct + fail: + msg: "The number of running druid tasks is less than the normal number!" + when: run_task_list.stdout_lines | intersect(init_task_list) |length != (init_task_list|length) + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml new file mode 100644 index 0000000..aaff071 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml @@ -0,0 +1,115 @@ +- name: Setting init directory + set_fact: + init_path: '{{ deploy_dir }}/druid/init/' + topology_path: '{{ deploy_dir }}/druid/topology/' + +- name: Remove files and directories + file: + path: "{{ init_path }}" + state: absent + ignore_errors: true + loop: + - "{{ init_path }}" + - "{{ topology_path }}" + +- name: Creatting init directory + file: + path: '{{ item }}' + state: directory + loop: + - "{{ init_path }}" + - "{{ topology_path }}" + - "{{ topology_path }}/tasks" + +- name: Setting hdfs_ip variable + set_fact: + hdfs_ip: "{{groups.hdfs[0]}}" + +- name: Unzipping segments.zip + unarchive: + src: 'files/cluster/segments.zip' + dest: '{{ deploy_dir }}/' + force: true + run_once: true + delegate_to: '{{ hdfs_ip }}' + +- name: Copying push segments to hdfs installation path + copy: + src: 'files/cluster/push_segments.sh' + dest: '{{ deploy_dir }}/push_segments.sh' + force: true + mode: 0755 + run_once: true + delegate_to: '{{ hdfs_ip }}' + +- name: Creating druid directory and putting segments to hdfs + shell: source /etc/profile && hadoop fs -mkdir -p /druid/segments + register: nums_out + run_once: true + delegate_to: '{{ hdfs_ip }}' + ignore_errors: yes + +- name: Putting segments to hdfs + shell: source /etc/profile && cd {{ deploy_dir }}/ && sh push_segments.sh + register: nums_out + run_once: true + delegate_to: '{{ hdfs_ip }}' + +- name: Copying mysql Client to /usr/bin/ + copy: + src: 'files/mysql' + dest: '/usr/bin/' + mode: 0755 + force: true + +- name: Copying init files to {{ init_path }} + copy: + src: 'files/cluster/' + dest: '{{ deploy_dir }}/druid/init' + force: true + +- name: Copying tasks template to {{ topology_path }} + template: + src: "{{ item }}" + dest: "{{ deploy_dir }}/druid/topology/tasks/{{ item | basename | regex_replace('.j2$', '') }}" + mode: "0644" + force: true + with_fileglob: "templates/tasks/*" + +- name: Inserting segments to mariadb + shell: 'mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} druid < {{ init_path }}/druid_segments-tsg.sql' + run_once: true + +- name: Copying script files + template: + src: '{{ item }}.j2' + dest: '{{ topology_path }}/{{ item }}' + mode: 0755 + with_items: + - 'druid_monitor.sh' + - 'supervisor-manager.sh' + +- name: Copying monitor template + template: + src: 'run_druid_monitor.j2' + dest: '/etc/cron.d/run_druid_monitor' + mode: 0644 + +- name: Submitting druid tasks + shell: cd {{ topology_path }} && sh supervisor-manager.sh startall + run_once: true + +- name: Deleting {{ deploy_dir }}/push_segments.sh + file: + path: "{{ deploy_dir }}/push_segments.sh" + state: absent + run_once: true + delegate_to: '{{ hdfs_ip }}' + +- name: Deleting {{ deploy_dir }}/segments.zip + file: + path: "{{ deploy_dir }}/segments.zip" + state: absent + run_once: true + delegate_to: '{{ hdfs_ip }}' + diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml new file mode 100644 index 0000000..9c10b77 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml @@ -0,0 +1,65 @@ +- name: Setting init directory + set_fact: + init_path: '{{ deploy_dir }}/druid/init/' + topology_path: '{{ deploy_dir }}/druid/topology/' + +- name: Creatting init directory + file: + path: '{{ item }}' + state: directory + loop: + - '{{ init_path }}' + - '{{ topology_path }}' + - '{{ topology_path }}/tasks' + - '{{ deploy_dir }}/druid/var/druid' + +- name: Unzipping segments.zip + unarchive: + src: 'files/standalone/segments.zip' + dest: '{{ deploy_dir }}/druid/var/druid/' + force: true + +- name: Copying mysql Client to /usr/bin/ + copy: + src: 'files/mysql' + dest: '/usr/bin/' + mode: 0755 + force: true + +- name: Copying init files to {{ init_path }} + copy: + src: 'files/standalone' + dest: '{{ init_path }}/' + force: true + +- name: Copying tasks template to {{ topology_path }} + template: + src: "{{ item }}" + dest: "{{ deploy_dir }}/druid/topology/tasks/{{ item | basename | regex_replace('.j2$', '') }}" + mode: "0644" + force: true + with_fileglob: "templates/tasks/*" + + +- name: Inserting segments to mariadb + shell: 'mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} druid < {{ init_path }}/standalone/druid_segments-tsg.sql' + run_once: true + +- name: Copying script files + template: + src: '{{ item }}.j2' + dest: '{{ topology_path }}/{{ item }}' + mode: 0755 + with_items: + - "druid_monitor.sh" + - "supervisor-manager.sh" + +- name: Copying monitor template + template: + src: 'run_druid_monitor.j2' + dest: '/etc/cron.d/run_druid_monitor' + mode: 0644 + +- name: Submitting druid tasks + shell: cd {{ topology_path }} && sh supervisor-manager.sh startall + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml new file mode 100644 index 0000000..35bf1ff --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml @@ -0,0 +1,21 @@ +- block: + - include: uninstall.yml + - include: "{{ playbook_name }}" + vars: + playbook_name: "{{ 'init-cluster.yml' if groups.druid | length > 1 else 'init-standalone.yml' }}" + - include: check.yml + when: inventory_hostname in groups['druid'][:2] and operation == 'install' + +- block: + - include: backup.yml + - include: uninstall.yml + - include: "{{ playbook_name }}" + vars: + playbook_name: "{{ 'init-cluster.yml' if groups.druid | length > 1 else 'init-standalone.yml' }}" + - include: check.yml + when: inventory_hostname in groups['druid'][:2] and (operation) == "upgrade" + +- block: + - include: uninstall.yml + when: inventory_hostname in groups['druid'][:2] and (operation) == "uninstall" + diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml new file mode 100644 index 0000000..ec46d7c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml @@ -0,0 +1,30 @@ +- name: Check {{ deploy_dir }}/druid/topology + stat: + path: "{{ deploy_dir }}/druid/topology" + register: directory_stat + +- block: + - name: Copying supervisor-manager.sh template + template: + src: 'supervisor-manager.sh.j2' + dest: '{{ deploy_dir }}/druid/topology/supervisor-manager.sh' + mode: 0755 + + - name: Stop all tasks + shell: cd {{ deploy_dir }}/druid/topology && sh supervisor-manager.sh terminateall + run_once: true + + - name: Get tasks status + shell: curl -s http://{{ vrrp_instance.default.virtual_ipaddress }}:8089/druid/indexer/v1/supervisor?state=true | jq 'map(select(.state = "RUNNING" ))' | jq 'length' + register: tasks_count + + - name: Check tasks status + debug: + msg: "{{ tasks_count.stdout }} tasks are not stopped" + failed_when: tasks_count.stdout > '0' + + - name: Delete {{ deploy_dir }}/druid/topology + file: + path: "{{ deploy_dir }}/druid/topology" + state: absent + when: directory_stat.stat.exists diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 new file mode 100644 index 0000000..e0c0444 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 @@ -0,0 +1,58 @@ +#!/bin/bash + +druid_ip={{ vrrp_instance.default.virtual_ipaddress }} +druid_port=8089 +druid_monitor_prom_file={{ deploy_dir }}/node-exporter/prom/druid_monitor.prom + +task_name=`cat tasklist` + +#获取supervisor状态信息 +function getSupervisorStatus(){ +druid_supervisor_healthy_num=`curl -G -d 'state=true' -s $druid_ip:$druid_port/druid/indexer/v1/supervisor | jq '[ .[] | .state] | length'` +echo druid_supervisor_healthy_num $druid_supervisor_healthy_num >> $druid_monitor_prom_file + +druid_supervisor_unhealthy_num=`curl -G -d 'state=true' -s $druid_ip:$druid_port/druid/indexer/v1/supervisor | jq '[ .[] | select(.state != "RUNNING")] | length'` +echo druid_supervisor_unhealthy_num $druid_supervisor_unhealthy_num >> $druid_monitor_prom_file +} + +#获取Druid 任务状态信息 +function getTaskStatus(){ +druid_index_running_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/runningTasks | jq '. | length'` +echo druid_index_running_task_num $druid_index_running_task_num > $druid_monitor_prom_file + +druid_index_waiting_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/waitingTasks | jq '. | length'` +echo druid_index_waiting_task_num $druid_index_waiting_task_num >> $druid_monitor_prom_file + +druid_index_pending_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/pendingTasks | jq '. | length'` +echo druid_index_pending_task_num $druid_index_pending_task_num >> $druid_monitor_prom_file + +druid_compact_pending_task_num=`curl -G -d 'type=compact' -s $druid_ip:$druid_port/druid/indexer/v1/pendingTasks | jq '. | length'` +echo druid_compact_pending_task_num $druid_compact_pending_task_num >> $druid_monitor_prom_file + +druid_compact_waiting_task_num=`curl -G -d 'type=compact' -s $druid_ip:$druid_port/druid/indexer/v1/waitingTasks | jq '. | length'` +echo druid_compact_waiting_task_num $druid_compact_waiting_task_num >> $druid_monitor_prom_file +} + +#校验正在运行的index数量，与启动时指定的index数量对比。 +#>=指定数值判断为正常 返回：0 +#反之异常 返回：1 +function checkIndexStatus(){ +druid_task_sum=0 +for var in ${task_name[@]}; +do + druid_task_num=`cat tasks/${var} | jq .ioConfig.taskCount` + druid_task_sum=`expr $druid_task_num + $druid_task_sum` +done + +druid_index_task_healthy_flag=0 +if [ $druid_index_running_task_num -lt $druid_task_sum ] +then + druid_index_task_healthy_flag=1 +fi + +echo druid_index_task_healthy_flag $druid_index_task_healthy_flag >> $druid_monitor_prom_file +} + +getTaskStatus +getSupervisorStatus +checkIndexStatus diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 new file mode 100644 index 0000000..bbffdc4 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 @@ -0,0 +1,28 @@ +#!/bin/bash + +base_dir=`dirname $0` +echo $base_dir + +#router节点IP +router_ip={{ groups.druid[0] }} + +common_task_name=`cat $base_dir/common_task.txt` +common_rule_file="$base_dir/common_data_rule.json" + +hot_task_name=`cat $base_dir/hot_task.txt` +hot_rule_file="$base_dir/hot_data_rule.json" + +for var in ${common_task_name[@]}; +do +curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @$common_rule_file http://${router_ip}:8088/druid/coordinator/v1/rules/${var} +echo "'${var}' 任务启动成功" +sleep 2 +done + +for var in ${hot_task_name[@]}; +do +curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @$hot_rule_file http://${router_ip}:8088/druid/coordinator/v1/rules/${var} +echo "'${var}' 任务启动成功" +sleep 2 +done + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 new file mode 100644 index 0000000..88da982 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 @@ -0,0 +1,75 @@ +#!/bin/bash + +base_dir=`dirname $0` +base_dir="$(cd "$base_dir" && pwd)" + +router_ip='{{ vrrp_instance.default.virtual_ipaddress }}' + +common_task_name=`cat $base_dir/tasklist` +common_rule_file="$base_dir/common_data_rule.json" + +hot_task_name=`cat $base_dir/hot_task.txt` +hot_rule_file="$base_dir/hot_data_rule.json" + +case $1 in + + resetAll) + for var in ${common_task_name[@]}; + do + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset + sleep 2 + done + + for var in ${hot_task_name[@]}; + do + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset + sleep 2 + done + ;; + + terminateAll) + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/terminateAll + ;; + + shutdownAllTasks) + for var in ${common_task_name[@]}; + do + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks + sleep 2 + done + + for var in ${hot_task_name[@]}; + do + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks + sleep 2 + done + ;; + + reset) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger reset " + exit 1 + fi + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/reset + ;; + + terminate) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger terminate " + exit 1 + fi + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/terminate + ;; + + shutdownTasks) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger shutdownTasks " + exit 1 + fi + curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/$2/shutdownAllTasks + ;; + *) + echo "Usage: supervisor-manger {resetAll|terminateAll|shutdownAllTasks}" + echo "Usage: supervisor-manger {reset|terminate|shutdownTasks} " + ;; +esac diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 new file mode 100644 index 0000000..d5fee98 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 @@ -0,0 +1,3 @@ +#Ansible: Check druid up to node_exporter +*/10 * * * * root cd {{ topology_path }} && sh druid_monitor.sh + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/s.bak b/tsg-olap/parcels/roles/initialization/druid/templates/s.bak new file mode 100644 index 0000000..ed547af --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/s.bak @@ -0,0 +1,124 @@ +#!/bin/bash +source /etc/profile + +DRUID_HOST='{{ vrrp_instance.default.virtual_ipaddress }}' +KAFKA_HOST="{% for host in groups['kafka'] %} +{{ host }}:9094{% if not loop.last %},{% endif %} +{% endfor %}" + +# 参数：1.操作 2.任务文件 +OPERATION=$1 + +TASK_LIST=( + index_kafka_application_protocol_stat.json + index_kafka_monitor_rule_hits.json + index_kafka_object_statistics.json + index_kafka_proxy_rule_hits.json + index_kafka_security_rule_hits.json + index_kafka_service_chaining_rule_hits.json + index_kafka_service_function_status.json + index_kafka_statistics_rule_hits.json + index_kafka_statistics_rule.json + index_kafka_sys_storage_log.json + index_kafka_top_client_ips.json + index_kafka_top_external_ips.json + index_kafka_top_internal_ips.json + index_kafka_top_server_domains.json + index_kafka_top_server_fqdns.json + index_kafka_top_server_ips.json + index_kafka_top_subscribers.json + index_kafka_traffic_general_stat.json + index_kafka_traffic_shaping_rule_hits.json + index_kafka_traffic_top_destination_ip_metrics_log.json +) + +if [ $# -lt 1 ]; then + echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" + echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] " + exit 1 +fi + +function start_task() { + local task_name=$1 + cp ./tasks/${task_name} ./ + + sed -i 's/kafkabootstrap/'$KAFKA_HOST'/' ${task_name} + + curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @${task_name} http://$DRUID_HOST:8089/druid/indexer/v1/supervisor + + if [ $? -eq "0" ]; then + echo "task:'${task_name}' Start-up success! " + rm -rf ./${task_name} + else + echo "task:'${task_name}' Start-up failure! " + exit 1 + fi + +} + +function reset_task() { + local task_name=$1 + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/reset" + sleep 2 +} + +function terminate_task() { + local task_name=$1 + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/terminate" +} + +function terminate_all() { + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/terminateAll" +} + +function shutdown_tasks() { + local task_name=$1 + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + local task_name=$1 + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/datasources/${task_name}/shutdownAllTasks" + sleep 2 +} + +case $OPERATION in +start) + start_task "$2" + ;; +startall) + for task in ${TASK_LIST[@]}; do + start_task "$task" + done + ;; +reset) + reset_task "$2" + ;; +resetall) + for task in ${TASK_LIST[@]}; do + reset_task "$task" + done + ;; +terminate) + terminate_task "$2" + ;; +terminateall) + terminate_all + ;; +shutdown) + shutdown_tasks "$2" + ;; +shutdownall) + for task in ${TASK_LIST[@]}; do + shutdown_tasks "$task" + done + ;; + +*) + echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" + echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] " + ;; +esac + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 new file mode 100644 index 0000000..e140a07 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 @@ -0,0 +1,103 @@ +#!/bin/bash +source /etc/profile + +DRUID_HOST='{{ vrrp_instance.default.virtual_ipaddress }}' +KAFKA_HOST="{% for host in groups['kafka'] %} +{{ host }}:9094{% if not loop.last %},{% endif %} +{% endfor %}" + +# 参数：1.操作 2.任务文件或目录 +OPERATION=$1 +filename=$(basename "$file") +TASK_FILE_LIST=$(find "./tasks" -name "*.json") + +if [ $# -lt 1 ]; then + echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" + echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] <./tasks/[task file]>" + exit 1 +fi + +function start_task() { + local task_file=$1 + local task_name=$(basename $task_file) + cp ${task_file} ./ + + sed -i 's/kafkabootstrap/'$KAFKA_HOST'/' ${task_name} + + curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @${task_name} http://$DRUID_HOST:8089/druid/indexer/v1/supervisor + + if [ $? -eq "0" ]; then + echo "task:'${task_name}' Start-up success! " + rm -rf ./${task_name} + else + echo "task:'${task_name}' Start-up failure! " + exit 1 + fi + +} + +function reset_task() { + local task_name=$(basename $1) + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/reset" + sleep 2 +} + +function terminate_task() { + local task_name=$(basename $1) + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/terminate" +} + +function terminate_all() { + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/terminateAll" +} + +function shutdown_tasks() { + local task_name=$(basename $1) + task_name="${task_name/index_kafka_/}" + task_name="${task_name/.json/}" + curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/datasources/${task_name}/shutdownAllTasks" + sleep 2 +} + +case $OPERATION in +start) + start_task "$2" + ;; +startall) + for task in ${TASK_FILE_LIST[@]}; do + start_task "$task" + done + ;; +reset) + reset_task "$2" + ;; +resetall) + for task in ${TASK_FILE_LIST[@]}; do + reset_task "$task" + done + ;; +terminate) + terminate_task "$2" + ;; +terminateall) + terminate_all + ;; +shutdown) + shutdown_tasks "$2" + ;; +shutdownall) + for task in ${TASK_FILE_LIST[@]}; do + shutdown_tasks "$task" + done + ;; + +*) + echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]" + echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] <./tasks/[task file]>" + ;; +esac + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json new file mode 100644 index 0000000..96f992a --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json @@ -0,0 +1,131 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "application_protocol_stat", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + "protocol_stack_id", + "app_name", + {"name": "vsys_id", "type": "long"} + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + {"name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id"}, + {"name": "data_center", "type": "path", "expr": "$.tags.data_center"}, + {"name": "device_group", "type": "path", "expr": "$.tags.device_group"}, + {"name": "device_id", "type": "path", "expr": "$.tags.device_id"}, + {"name": "protocol_stack_id", "type": "path", "expr": "$.tags.protocol_stack_id"}, + {"name": "app_name", "type": "path", "expr": "$.tags.app_name"}, + {"name": "sessions", "type": "path", "expr": "$.fields.sessions"}, + {"name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes"}, + {"name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes"}, + {"name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts"}, + {"name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts"}, + {"name": "c2s_pkts", "type": "path", "expr": "$.fields.c2s_pkts"}, + {"name": "s2c_pkts", "type": "path", "expr": "$.fields.s2c_pkts"}, + {"name": "c2s_bytes", "type": "path", "expr": "$.fields.c2s_bytes"}, + {"name": "s2c_bytes", "type": "path", "expr": "$.fields.s2c_bytes"}, + {"name": "c2s_fragments", "type": "path", "expr": "$.fields.c2s_fragments"}, + {"name": "s2c_fragments", "type": "path", "expr": "$.fields.s2c_fragments"}, + {"name": "c2s_tcp_lost_bytes", "type": "path", "expr": "$.fields.c2s_tcp_lost_bytes"}, + {"name": "s2c_tcp_lost_bytes", "type": "path", "expr": "$.fields.s2c_tcp_lost_bytes"}, + {"name": "c2s_tcp_ooorder_pkts", "type": "path", "expr": "$.fields.c2s_tcp_ooorder_pkts"}, + {"name": "s2c_tcp_ooorder_pkts", "type": "path", "expr": "$.fields.s2c_tcp_ooorder_pkts"}, + {"name": "c2s_tcp_retransmitted_pkts", "type": "path", "expr": "$.fields.c2s_tcp_retransmitted_pkts"}, + {"name": "s2c_tcp_retransmitted_pkts", "type": "path", "expr": "$.fields.s2c_tcp_retransmitted_pkts"}, + {"name": "c2s_tcp_retransmitted_bytes", "type": "path", "expr": "$.fields.c2s_tcp_retransmitted_bytes"}, + {"name": "s2c_tcp_retransmitted_bytes", "type": "path", "expr": "$.fields.s2c_tcp_retransmitted_bytes"} + ] + } + } + }, + "metricsSpec": [ + {"type": "longSum", "name": "sessions", "fieldName": "sessions"}, + {"type": "longSum", "name": "c2s_pkts", "fieldName": "c2s_pkts"}, + {"type": "longSum", "name": "s2c_pkts", "fieldName": "s2c_pkts"}, + {"type": "longSum", "name": "c2s_bytes", "fieldName": "c2s_bytes"}, + {"type": "longSum", "name": "s2c_bytes", "fieldName": "s2c_bytes"}, + {"type": "longSum", "name": "c2s_fragments", "fieldName": "c2s_fragments"}, + {"type": "longSum", "name": "s2c_fragments", "fieldName": "s2c_fragments"}, + {"type": "longSum", "name": "c2s_tcp_lost_bytes", "fieldName": "c2s_tcp_lost_bytes"}, + {"type": "longSum", "name": "s2c_tcp_lost_bytes", "fieldName": "s2c_tcp_lost_bytes"}, + {"type": "longSum", "name": "c2s_tcp_ooorder_pkts", "fieldName": "c2s_tcp_ooorder_pkts"}, + {"type": "longSum", "name": "s2c_tcp_ooorder_pkts", "fieldName": "s2c_tcp_ooorder_pkts"}, + {"type": "longSum", "name": "c2s_tcp_retransmitted_pkts", "fieldName": "c2s_tcp_retransmitted_pkts"}, + {"type": "longSum", "name": "s2c_tcp_retransmitted_pkts", "fieldName": "s2c_tcp_retransmitted_pkts"}, + {"type": "longSum", "name": "c2s_tcp_retransmitted_bytes", "fieldName": "c2s_tcp_retransmitted_bytes"}, + {"type": "longSum", "name": "s2c_tcp_retransmitted_bytes", "fieldName": "s2c_tcp_retransmitted_bytes"}, + {"type": "longSum", "name": "in_bytes", "fieldName": "in_bytes"}, + {"type": "longSum", "name": "out_bytes", "fieldName": "out_bytes"}, + {"type": "longSum", "name": "in_pkts", "fieldName": "in_pkts"}, + {"type": "longSum", "name": "out_pkts", "fieldName": "out_pkts"} + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT15S"}, + "rollup": true + }, + "transformSpec": { + "transforms": [ + {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "c2s_pkts", "expression": "nvl(c2s_pkts, 0)"}, + {"type": "expression", "name": "s2c_pkts", "expression": "nvl(s2c_pkts, 0)"}, + {"type": "expression", "name": "c2s_bytes", "expression": "nvl(c2s_bytes, 0)"}, + {"type": "expression", "name": "s2c_bytes", "expression": "nvl(s2c_bytes, 0)"}, + {"type": "expression", "name": "c2s_fragments", "expression": "nvl(c2s_fragments, 0)"}, + {"type": "expression", "name": "s2c_fragments", "expression": "nvl(s2c_fragments, 0)"}, + {"type": "expression", "name": "c2s_tcp_lost_bytes", "expression": "nvl(c2s_tcp_lost_bytes, 0)"}, + {"type": "expression", "name": "s2c_tcp_lost_bytes", "expression": "nvl(s2c_tcp_lost_bytes, 0)"}, + {"type": "expression", "name": "c2s_tcp_ooorder_pkts", "expression": "nvl(c2s_tcp_ooorder_pkts, 0)"}, + {"type": "expression", "name": "s2c_tcp_ooorder_pkts", "expression": "nvl(s2c_tcp_ooorder_pkts, 0)"}, + {"type": "expression", "name": "c2s_tcp_retransmitted_pkts", "expression": "nvl(c2s_tcp_retransmitted_pkts, 0)"}, + {"type": "expression", "name": "s2c_tcp_retransmitted_pkts", "expression": "nvl(s2c_tcp_retransmitted_pkts, 0)"}, + {"type": "expression", "name": "c2s_tcp_retransmitted_bytes", "expression": "nvl(c2s_tcp_retransmitted_bytes, 0)"}, + {"type": "expression", "name": "s2c_tcp_retransmitted_bytes", "expression": "nvl(s2c_tcp_retransmitted_bytes, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter": { + "type": "selector", + "dimension": "name", + "value": "application_protocol_stat" + } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "NETWORK-TRAFFIC-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json new file mode 100644 index 0000000..5c43f46 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json @@ -0,0 +1,83 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "monitor_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + {"name": "rule_id","type": "long"}, + {"name": "action","type": "long"} + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, + { "name": "action", "type": "path", "expr": "$.tags.action" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "monitor_rule_hits" } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json new file mode 100644 index 0000000..84dd11b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json @@ -0,0 +1,95 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "object_statistics", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + {"name": "template_id","type": "long"}, + {"name": "chart_id","type": "long"}, + {"name": "version","type": "long"}, + "object_type", + {"name": "object_id","type": "long"}, + {"name": "item_id","type": "long"} + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "bytes", "type": "path", "expr": "$.fields.bytes" }, + { "name": "new_in_sessions", "type": "path", "expr": "$.fields.new_in_sessions" }, + { "name": "new_out_sessions", "type": "path", "expr": "$.fields.new_out_sessions" }, + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "template_id", "type": "path", "expr": "$.tags.template_id" }, + { "name": "chart_id", "type": "path", "expr": "$.tags.chart_id" }, + { "name": "version", "type": "path", "expr": "$.tags.version" }, + { "name": "object_type", "type": "path", "expr": "$.tags.object_type" }, + { "name": "object_id", "type": "path", "expr": "$.tags.object_id" }, + { "name": "item_id", "type": "path", "expr": "$.tags.item_id" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" }, + { "type" : "longSum", "name" : "new_in_sessions", "fieldName" : "new_in_sessions" }, + { "type" : "longSum", "name" : "new_out_sessions", "fieldName" : "new_out_sessions" }, + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT15S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"}, + {"type": "expression", "name": "new_in_sessions", "expression": "nvl(new_in_sessions, 0)"}, + {"type": "expression", "name": "new_out_sessions", "expression": "nvl(new_out_sessions, 0)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "object_statistics" } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "OBJECT-STATISTICS-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json new file mode 100644 index 0000000..ab97fb1 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json @@ -0,0 +1,90 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "proxy_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { "name": "vsys_id", "type": "long" }, + { "name": "rule_id", "type": "long" }, + { "name": "pinning_status", "type": "string" }, + { "name": "action", "type": "long" }, + { "name": "sub_action", "type": "string" } + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "pinning_status", "type": "path", "expr": "$.tags.pinning_status" }, + { "name": "action", "type": "path", "expr": "$.tags.action" }, + { "name": "sub_action", "type": "path", "expr": "$.tags.sub_action" }, + { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "proxy_rule_hits" } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json new file mode 100644 index 0000000..957923d --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json @@ -0,0 +1,83 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "security_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + {"name": "rule_id","type": "long"}, + {"name": "action","type": "long"} + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, + { "name": "action", "type": "path", "expr": "$.tags.action" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "security_rule_hits" } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json new file mode 100644 index 0000000..6f3e3bd --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json @@ -0,0 +1,84 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "service_chaining_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { "name": "vsys_id", "type": "long" }, + { "name": "rule_id", "type": "long" }, + { "name": "sff_profile_id", "type": "long" }, + { "name": "sf_profile_id", "type": "long" } + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, + { "name": "sff_profile_id", "type": "path", "expr": "$.tags.sff_profile_id" }, + { "name": "sf_profile_id", "type": "path", "expr": "$.tags.sf_profile_id" }, + { "name": "sent_pkts", "type": "path", "expr": "$.fields.sent_pkts" }, + { "name": "sent_bytes", "type": "path", "expr": "$.fields.sent_bytes" }, + { "name": "recv_pkts", "type": "path", "expr": "$.fields.recv_pkts" }, + { "name": "recv_bytes", "type": "path", "expr": "$.fields.recv_bytes" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sent_pkts", "fieldName" : "sent_pkts" }, + { "type" : "longSum", "name" : "sent_bytes", "fieldName" : "sent_bytes" }, + { "type" : "longSum", "name" : "recv_pkts", "fieldName" : "recv_pkts" }, + { "type" : "longSum", "name" : "recv_bytes", "fieldName" : "recv_bytes" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "sent_pkts", "expression": "nvl(sent_pkts, 0)"}, + {"type": "expression", "name": "sent_bytes", "expression": "nvl(sent_bytes, 0)"}, + {"type": "expression", "name": "recv_pkts", "expression": "nvl(recv_pkts, 0)"}, + {"type": "expression", "name": "recv_bytes", "expression": "nvl(recv_bytes, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "service_chaining_rule_hits" } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json new file mode 100644 index 0000000..61e5b12 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json @@ -0,0 +1,74 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "service_function_status", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { "name": "vsys_id", "type": "long" }, + { "name": "sf_profile_id", "type": "long" } + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "sf_profile_id", "type": "path", "expr": "$.tags.sf_profile_id" }, + { "name": "sf_status", "type": "path", "expr": "$.fields.sf_status" }, + { "name": "sf_latency_us", "type": "path", "expr": "$.fields.sf_latency_us" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longMax", "name" : "sf_latency_us", "fieldName" : "sf_latency_us" }, + { "type" : "longMax", "name" : "sf_status", "fieldName" : "sf_status" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "sf_latency_us", "expression": "nvl(sf_latency_us, 0)"}, + {"type": "expression", "name": "sf_status", "expression": "nvl(sf_status, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "service_function_status" } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json new file mode 100644 index 0000000..8c15a37 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json @@ -0,0 +1,367 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "statistics_rule", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { + "name": "vsys_id", + "type": "long" + }, + { + "name": "rule_id", + "type": "long" + }, + { + "name": "template_id", + "type": "long" + }, + { + "name": "chart_id", + "type": "long" + }, + { + "name": "version", + "type": "long" + }, + "client_ip_object", + "server_ip_object", + "fqdn_category", + "client_ip", + "server_ip", + "server_fqdn", + "server_domain", + "application" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { + "name": "vsys_id", + "type": "path", + "expr": "$.tags.vsys_id" + }, + { + "name": "data_center", + "type": "path", + "expr": "$.tags.data_center" + }, + { + "name": "device_group", + "type": "path", + "expr": "$.tags.device_group" + }, + { + "name": "device_id", + "type": "path", + "expr": "$.tags.device_id" + }, + { + "name": "rule_id", + "type": "path", + "expr": "$.tags.rule_id" + }, + { + "name": "template_id", + "type": "path", + "expr": "$.tags.template_id" + }, + { + "name": "chart_id", + "type": "path", + "expr": "$.tags.chart_id" + }, + { + "name": "version", + "type": "path", + "expr": "$.tags.version" + }, + { + "name": "client_ip_object", + "type": "path", + "expr": "$.tags.client_ip_object" + }, + { + "name": "server_ip_object", + "type": "path", + "expr": "$.tags.server_ip_object" + }, + { + "name": "fqdn_category", + "type": "path", + "expr": "$.tags.fqdn_category" + }, + { + "name": "client_ip", + "type": "path", + "expr": "$.tags.client_ip" + }, + { + "name": "server_ip", + "type": "path", + "expr": "$.tags.server_ip" + }, + { + "name": "server_fqdn", + "type": "path", + "expr": "$.tags.server_fqdn" + }, + { + "name": "server_domain", + "type": "path", + "expr": "$.tags.server_domain" + }, + { + "name": "application", + "type": "path", + "expr": "$.tags.application" + }, + { + "name": "in_bytes", + "type": "path", + "expr": "$.fields.in_bytes" + }, + { + "name": "out_bytes", + "type": "path", + "expr": "$.fields.out_bytes" + }, + { + "name": "bytes", + "type": "path", + "expr": "$.fields.bytes" + }, + { + "name": "new_c2s_flows", + "type": "path", + "expr": "$.fields.new_c2s_flows" + }, + { + "name": "new_s2c_flows", + "type": "path", + "expr": "$.fields.new_s2c_flows" + }, + { + "name": "new_in_sessions", + "type": "path", + "expr": "$.fields.new_in_sessions" + }, + { + "name": "new_out_sessions", + "type": "path", + "expr": "$.fields.new_out_sessions" + }, + { + "name": "sessions", + "type": "path", + "expr": "$.fields.sessions" + }, + { + "name": "new_unestablished_sessions", + "type": "path", + "expr": "$.fields.new_unestablished_sessions" + }, + { + "name": "syn_pkts", + "type": "path", + "expr": "$.fields.syn_pkts" + }, + { + "name": "session_identifier_sketch", + "type": "path", + "expr": "$.fields.session_identifier_sketch" + }, + { + "name": "client_ip_sketch", + "type": "path", + "expr": "$.fields.client_ip_sketch" + }, + { + "name": "server_ip_sketch", + "type": "path", + "expr": "$.fields.server_ip_sketch" + }, + { + "name": "in_latency_ms_sketch", + "type": "path", + "expr": "$.fields.in_latency_ms_sketch" + }, + { + "name": "out_latency_ms_sketch", + "type": "path", + "expr": "$.fields.out_latency_ms_sketch" + }, + { + "name": "latency_ms_sketch", + "type": "path", + "expr": "$.fields.latency_ms_sketch" + } + ] + } + } + }, + "metricsSpec": [ + { + "type": "HLLDSketchMerge", + "name": "session_identifier_sketch", + "fieldName": "session_identifier_sketch" + }, + { + "type": "HLLDSketchMerge", + "name": "client_ip_sketch", + "fieldName": "client_ip_sketch" + },{ + "type": "HLLDSketchMerge", + "name": "server_ip_sketch", + "fieldName": "server_ip_sketch" + }, + { + "type": "HdrHistogramSketchMerge", + "name": "in_latency_ms_sketch", + "fieldName": "in_latency_ms_sketch", + "lowestDiscernibleValue": "1", + "highestTrackableValue": "2", + "numberOfSignificantValueDigits": "3", + "autoResize": "true" + }, + { + "type": "HdrHistogramSketchMerge", + "name": "out_latency_ms_sketch", + "fieldName": "out_latency_ms_sketch", + "lowestDiscernibleValue": "1", + "highestTrackableValue": "2", + "numberOfSignificantValueDigits": "3", + "autoResize": "true" + }, + { + "type": "HdrHistogramSketchMerge", + "name": "latency_ms_sketch", + "fieldName": "latency_ms_sketch", + "lowestDiscernibleValue": "1", + "highestTrackableValue": "2", + "numberOfSignificantValueDigits": "3", + "autoResize": "true" + }, + { + "type": "longSum", + "name": "in_bytes", + "fieldName": "in_bytes" + }, + { + "type": "longSum", + "name": "out_bytes", + "fieldName": "out_bytes" + }, + { + "type": "longSum", + "name": "bytes", + "fieldName": "bytes" + }, + { + "type": "longSum", + "name": "new_c2s_flows", + "fieldName": "new_c2s_flows" + }, + { + "type": "longSum", + "name": "new_s2c_flows", + "fieldName": "new_s2c_flows" + }, + { + "type": "longSum", + "name": "new_in_sessions", + "fieldName": "new_in_sessions" + }, + { + "type": "longSum", + "name": "new_out_sessions", + "fieldName": "new_out_sessions" + }, + { + "type": "longSum", + "name": "sessions", + "fieldName": "sessions" + }, + { + "type": "longSum", + "name": "new_unestablished_sessions", + "fieldName": "new_unestablished_sessions" + }, + { + "type": "longSum", + "name": "syn_pkts", + "fieldName": "syn_pkts" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "period", + "period": "PT15S" + }, + "rollup": true + }, + "transformSpec": { + "transforms": [ + {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"}, + {"type": "expression", "name": "client_ip_object", "expression": "STRING_TO_ARRAY(client_ip_object,',')"}, + {"type": "expression", "name": "server_ip_object", "expression": "STRING_TO_ARRAY(server_ip_object,',')"}, + {"type": "expression", "name": "fqdn_category", "expression": "STRING_TO_ARRAY(fqdn_category,',')"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"}, + {"type": "expression", "name": "new_c2s_flows", "expression": "nvl(new_c2s_flows, 0)"}, + {"type": "expression", "name": "new_s2c_flows", "expression": "nvl(new_s2c_flows, 0)"}, + {"type": "expression", "name": "new_in_sessions", "expression": "nvl(new_in_sessions, 0)"}, + {"type": "expression", "name": "new_out_sessions", "expression": "nvl(new_out_sessions, 0)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "new_unestablished_sessions", "expression": "nvl(new_unestablished_sessions, 0)"}, + {"type": "expression", "name": "syn_pkts", "expression": "nvl(syn_pkts, 0)"} + ], + "filter": { + "type": "selector", + "dimension": "name", + "value": "statistics_rule" + } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "maxRowsInMemory": 400000, + "maxBytesInMemory": -1, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "STATISTICS-RULE-METRIC", + "taskCount": 2, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + }, + "context":{ + "druid.indexer.runner.javaOpts": "{{ druid.index_kafka_statistics_rule['druid.indexer.runner.javaOpts'] }}" + } +} + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json new file mode 100644 index 0000000..f431e0f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json @@ -0,0 +1,159 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "statistics_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { + "name": "vsys_id", + "type": "long" + }, + { + "name": "rule_id", + "type": "long" + } + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { + "name": "vsys_id", + "type": "path", + "expr": "$.tags.vsys_id" + }, + { + "name": "data_center", + "type": "path", + "expr": "$.tags.data_center" + }, + { + "name": "device_group", + "type": "path", + "expr": "$.tags.device_group" + }, + { + "name": "device_id", + "type": "path", + "expr": "$.tags.device_id" + }, + { + "name": "rule_id", + "type": "path", + "expr": "$.tags.rule_id" + }, + { + "name": "in_bytes", + "type": "path", + "expr": "$.fields.in_bytes" + }, + { + "name": "out_bytes", + "type": "path", + "expr": "$.fields.out_bytes" + }, + { + "name": "hit_count", + "type": "path", + "expr": "$.fields.hit_count" + }, + { + "name": "in_pkts", + "type": "path", + "expr": "$.fields.in_pkts" + }, + { + "name": "out_pkts", + "type": "path", + "expr": "$.fields.out_pkts" + } + ] + } + } + }, + "metricsSpec": [ + { + "type": "longSum", + "name": "in_bytes", + "fieldName": "in_bytes" + }, + { + "type": "longSum", + "name": "out_bytes", + "fieldName": "out_bytes" + }, + { + "type": "longSum", + "name": "hit_count", + "fieldName": "hit_count" + }, + { + "type": "longSum", + "name": "in_pkts", + "fieldName": "in_pkts" + }, + { + "type": "longSum", + "name": "out_pkts", + "fieldName": "out_pkts" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": { + "type": "period", + "period": "PT1S" + }, + "rollup": true + }, + "transformSpec": { + "transforms": [ + {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter": { + "type": "selector", + "dimension": "name", + "value": "statistics_rule_hits" + } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json new file mode 100644 index 0000000..2b405c2 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json @@ -0,0 +1,52 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sys_storage_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "log_type", + "data_center", + { "name": "used_size", "type": "long" }, + { "name": "max_size", "type": "long" }, + { "name": "aggregate_size", "type": "long" }, + { "name": "last_storage", "type": "long" } + ] + } + } + }, + "metricsSpec" : [ + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "SYS-STORAGE", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json new file mode 100644 index 0000000..4a8749f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json @@ -0,0 +1,104 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_client_ips", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "client_ip", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "client_ip", "type": "path", "expr": "$.tags.client_ip" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"packets_top_client_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_client_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_client_ip" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json new file mode 100644 index 0000000..5ff580e --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_external_ips", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "external_ip", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "external_ip", "type": "path", "expr": "$.tags.external_ip" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_external_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_external_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"packets_top_external_ip" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json new file mode 100644 index 0000000..8767165 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_internal_ips", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "internal_ip", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "internal_ip", "type": "path", "expr": "$.tags.internal_ip" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_internal_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_internal_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"packets_top_internal_ip" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json new file mode 100644 index 0000000..bd5b144 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_server_domains", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "domain", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "domain", "type": "path", "expr": "$.tags.domain" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"packets_top_server_domain" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_server_domain" + }, + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_server_domain" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json new file mode 100644 index 0000000..4bc42af --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_server_fqdns", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "fqdn", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "fqdn", "type": "path", "expr": "$.tags.fqdn" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_server_fqdn" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_server_fqdn" + }, + { + "type":"selector", + "dimension":"name", + "value":"packets_top_server_fqdn" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json new file mode 100644 index 0000000..1f05adf --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_server_ips", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "server_ip", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "server_ip", "type": "path", "expr": "$.tags.server_ip" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_server_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_server_ip" + }, + { + "type":"selector", + "dimension":"name", + "value":"packets_top_server_ip" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json new file mode 100644 index 0000000..8bc5e37 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_subscribers", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + "subscriber_id", + "metric" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "subscriber_id", "type": "path", "expr": "$.tags.subscriber_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "name", "type": "path", "expr": "$.name" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"}, + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"} + ], + "filter":{ + "type":"or","fields":[ + { + "type":"selector", + "dimension":"name", + "value":"sessions_top_subscriber_id" + }, + { + "type":"selector", + "dimension":"name", + "value":"bytes_top_subscriber_id" + }, + { + "type":"selector", + "dimension":"name", + "value":"packets_top_subscriber_id" + } + ] + } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + + + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json new file mode 100644 index 0000000..6013977 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json @@ -0,0 +1,105 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_general_stat", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "device_group", + "data_center", + { "name": "vsys_id", "type": "long" } + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "sessions", "type": "path", "expr": "$.fields.sessions" }, + { "name": "active_sessions", "type": "path", "expr": "$.fields.active_sessions" }, + { "name": "closed_sessions", "type": "path", "expr": "$.fields.closed_sessions" }, + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "asymmetric_c2s_flows", "type": "path", "expr": "$.fields.asymmetric_c2s_flows" }, + { "name": "asymmetric_s2c_flows", "type": "path", "expr": "$.fields.asymmetric_s2c_flows" }, + { "name": "tcp_syn_pkts", "type": "path", "expr": "$.fields.tcp_syn_pkts" }, + { "name": "tcp_ack_pkts", "type": "path", "expr": "$.fields.tcp_ack_pkts" }, + { "name": "tcp_fin_pkts", "type": "path", "expr": "$.fields.tcp_fin_pkts" }, + { "name": "tcp_rst_pkts", "type": "path", "expr": "$.fields.tcp_rst_pkts" } + ] + } + } + }, + "metricsSpec":[ + { "type":"longSum", "name":"sessions", "fieldName":"sessions" }, + { "type":"longMax", "name":"active_sessions", "fieldName":"active_sessions" }, + { "type":"longSum", "name":"closed_sessions", "fieldName":"closed_sessions" }, + { "type":"longSum", "name":"in_bytes", "fieldName":"in_bytes" }, + { "type":"longSum", "name":"out_bytes", "fieldName":"out_bytes" }, + { "type":"longSum", "name":"in_pkts", "fieldName":"in_pkts" }, + { "type":"longSum", "name":"out_pkts", "fieldName":"out_pkts" }, + { "type":"longSum", "name":"asymmetric_c2s_flows", "fieldName":"asymmetric_c2s_flows" }, + { "type":"longSum", "name":"asymmetric_s2c_flows", "fieldName":"asymmetric_s2c_flows" }, + { "type":"longSum", "name":"tcp_syn_pkts", "fieldName":"tcp_syn_pkts" }, + { "type":"longSum", "name":"tcp_ack_pkts", "fieldName":"tcp_ack_pkts" }, + { "type":"longSum", "name":"tcp_fin_pkts", "fieldName":"tcp_fin_pkts" }, + { "type":"longSum", "name":"tcp_rst_pkts", "fieldName":"tcp_rst_pkts" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}, + {"type": "expression", "name": "active_sessions", "expression": "nvl(active_sessions, 0)"}, + {"type": "expression", "name": "closed_sessions", "expression": "nvl(closed_sessions, 0)"}, + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}, + {"type": "expression", "name": "asymmetric_c2s_flows", "expression": "nvl(asymmetric_c2s_flows, 0)"}, + {"type": "expression", "name": "asymmetric_s2c_flows", "expression": "nvl(asymmetric_s2c_flows, 0)"}, + {"type": "expression", "name": "tcp_syn_pkts", "expression": "nvl(tcp_syn_pkts, 0)"}, + {"type": "expression", "name": "tcp_ack_pkts", "expression": "nvl(tcp_ack_pkts, 0)"}, + {"type": "expression", "name": "tcp_fin_pkts", "expression": "nvl(tcp_fin_pkts, 0)"}, + {"type": "expression", "name": "tcp_rst_pkts", "expression": "nvl(tcp_rst_pkts, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "traffic_general_stat" } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "NETWORK-TRAFFIC-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json new file mode 100644 index 0000000..a5a644d --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json @@ -0,0 +1,101 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_shaping_rule_hits", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp_ms", + "format": "millis" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "vsys_id","type": "long"}, + "device_id", + "device_group", + "data_center", + {"name": "rule_id","type": "long"}, + {"name": "profile_id","type": "long"}, + {"name": "priority","type": "long"} + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" }, + { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" }, + { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" }, + { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }, + { "name": "in_drop_pkts", "type": "path", "expr": "$.fields.in_drop_pkts" }, + { "name": "out_drop_pkts", "type": "path", "expr": "$.fields.out_drop_pkts" }, + { "name": "in_max_latency_us", "type": "path", "expr": "$.fields.in_max_latency_us" }, + { "name": "out_max_latency_us", "type": "path", "expr": "$.fields.out_max_latency_us" }, + { "name": "in_queue_len", "type": "path", "expr": "$.fields.in_queue_len" }, + { "name": "out_queue_len", "type": "path", "expr": "$.fields.out_queue_len" }, + { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" }, + { "name": "data_center", "type": "path", "expr": "$.tags.data_center" }, + { "name": "device_group", "type": "path", "expr": "$.tags.device_group" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" }, + { "name": "profile_id", "type": "path", "expr": "$.tags.profile_id" }, + { "name": "priority", "type": "path", "expr": "$.tags.priority" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" }, + { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" }, + { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" }, + { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }, + { "type" : "longSum", "name" : "in_drop_pkts", "fieldName" : "in_drop_pkts" }, + { "type" : "longSum", "name" : "out_drop_pkts", "fieldName" : "out_drop_pkts" }, + { "type" : "longMax", "name" : "in_max_latency_us", "fieldName" : "in_max_latency_us" }, + { "type" : "longMax", "name" : "out_max_latency_us", "fieldName" : "out_max_latency_us" }, + { "type" : "longMax", "name" : "in_queue_len", "fieldName" : "in_queue_len" }, + { "type" : "longMax", "name" : "out_queue_len", "fieldName" : "out_queue_len" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1S"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"}, + {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"}, + {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"}, + {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}, + {"type": "expression", "name": "in_drop_pkts", "expression": "nvl(in_drop_pkts, 0)"}, + {"type": "expression", "name": "out_drop_pkts", "expression": "nvl(out_drop_pkts, 0)"}, + {"type": "expression", "name": "in_max_latency_us", "expression": "nvl(in_max_latency_us, 0)"}, + {"type": "expression", "name": "out_max_latency_us", "expression": "nvl(out_max_latency_us, 0)"}, + {"type": "expression", "name": "in_queue_len", "expression": "nvl(in_queue_len, 0)"}, + {"type": "expression", "name": "out_queue_len", "expression": "nvl(out_queue_len, 0)"} + ], + "filter": { "type": "selector", "dimension": "name", "value": "traffic_shaping_rule_hits" } + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "POLICY-RULE-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} + diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json new file mode 100644 index 0000000..5541452 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json @@ -0,0 +1,61 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_top_destination_ip_metrics_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "sketch_start_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "destination_ip", + "attack_type", + { "name": "vsys_id", "type": "long" }, + { "type": "long", "name": "partition_num" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_rate","fieldName" : "session_rate"}, + { "type" : "longSum", "name" : "packet_rate", "fieldName" : "packet_rate" }, + { "type" : "longSum", "name" : "bit_rate", "fieldName" : "bit_rate" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT10M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression","name": "vsys_id","expression": "nvl(vsys_id,1)"}, + {"type": "expression", "name": "session_rate", "expression": "nvl(session_rate, 0)"}, + {"type": "expression", "name": "packet_rate", "expression": "nvl(packet_rate, 0)"}, + {"type": "expression", "name": "bit_rate", "expression": "nvl(bit_rate, 0)"} + ] + } + }, + "tuningConfig": { + "type": "kafka", + "resetOffsetAutomatically": true, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-TOP-DESTINATION-IP-METRIC", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "sasl.mechanism": "PLAIN", + "security.protocol": "SASL_PLAINTEXT", + "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";" + } + } +} diff --git a/tsg-olap/parcels/roles/initialization/druid/vars/main.yml b/tsg-olap/parcels/roles/initialization/druid/vars/main.yml new file mode 100644 index 0000000..3fc948a --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/druid/vars/main.yml @@ -0,0 +1,11 @@ +container_name: druid + +backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/{{ container_name }}" + +backup_items: + - conf + - docker-compose.yml + - init + - monitor + - topology + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml new file mode 100644 index 0000000..2c5336b --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml @@ -0,0 +1,28 @@ +- name: Create buckets + uri: + url: "http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/{{ item }}" + method: PUT + headers: + token: "{{ hos_token }}" + return_content: yes + retries: 3 + delay: 3 + loop: "{{ init_buckets }}" + run_once: true + +- name: Check if buckets exist + uri: + url: "http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/" + method: GET + headers: + token: "{{ hos_token }}" + return_content: yes + register: bucket_info + retries: 3 + delay: 3 + +- name: Check create bucket result + fail: + msg: "{{ item }} was not created successfully. Please retry." + when: "'{{ item }}' not in bucket_info.content" + loop: "{{ init_buckets }}" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml new file mode 100644 index 0000000..125d374 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml @@ -0,0 +1,5 @@ +- include: init.yml + when: operation == 'install' + +- include: upgrade.yml + when: operation == 'upgrade' diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml new file mode 100644 index 0000000..cd21505 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml @@ -0,0 +1,2 @@ +--- + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml new file mode 100644 index 0000000..9203abd --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml @@ -0,0 +1,6 @@ +init_buckets: + - troubleshooting_file_bucket + - assessment_file_bucket + - knowledge_base_bucket + - traffic_file_bucket + - report_snapshot_bucket diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/mysql b/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/mysql new file mode 100644 index 0000000..a4f1e76 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/mysql differ diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/schedule-service/dos-baseline/analyse/analyse-baseines.jar b/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/schedule-service/dos-baseline/analyse/analyse-baseines.jar new file mode 100644 index 0000000..0378b99 Binary files /dev/null and b/tsg-olap/parcels/roles/initialization/galaxy-job-service/files/schedule-service/dos-baseline/analyse/analyse-baseines.jar differ diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml new file mode 100644 index 0000000..3a66bc3 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml @@ -0,0 +1,51 @@ +- name: Check if admin backup directory exists + stat: + path: "{{ backup_path }}/{{ admin_container_name }}" + register: exist_status + +- name: Backup directories + block: + - name: Creat backup directory + file: + state: directory + path: "{{ backup_path }}/{{ admin_container_name }}" + + - name: Copying directories to backup + copy: + src: "{{ deploy_dir }}/{{ admin_container_name }}/{{ item }}" + dest: "{{ backup_path }}/{{ admin_container_name }}/" + remote_src: true + loop: "{{ backup_items }}" + + rescue: + - name: Remove backup directory on failure + file: + path: "{{ backup_path }}/{{ admin_container_name }}" + state: absent + when: exist_status.stat.exists == false + +- name: Check if executor backup directory exists + stat: + path: "{{ backup_path }}/{{ executor_container_name }}" + register: exist_status + +- name: Backup directories + block: + - name: Creat backup directory + file: + state: directory + path: "{{ backup_path }}/{{ executor_container_name }}" + + - name: Copying directories to backup + copy: + src: "{{ deploy_dir }}/{{ executor_container_name }}/{{ item }}" + dest: "{{ backup_path }}/{{ executor_container_name }}/" + remote_src: true + loop: "{{ backup_items }}" + + rescue: + - name: Remove backup directory on failure + file: + path: "{{ backup_path }}/{{ executor_container_name }}" + state: absent + when: exist_status.stat.exists == false diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml new file mode 100644 index 0000000..14e1d40 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml @@ -0,0 +1,12 @@ +- name: Check the health status of galaxy-job-admin + uri: + url: http://{{ vrrp_instance.default.virtual_ipaddress }}:8181/xxl-job-admin/login + method: POST + body_format: form-urlencoded + body: + userName: admin + pin: galaxy_2019 + +- name: Check that the galaxy-job-admin is accessible + uri: + url: "http://{{ vrrp_instance.default.virtual_ipaddress }}:8181/xxl-job-admin/testCase/logPolicyTest" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml new file mode 100644 index 0000000..14cfe28 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml @@ -0,0 +1,52 @@ +- name: Set container_name + set_fact: + container_name: "{{ admin_container_name }}" + image_name: "{{ admin_image_name }}" + image_tag: "{{ admin_image_tag }}" + +- name: Creating directory + file: + state: directory + path: '{{ deploy_dir }}/{{ container_name }}/config' + +- name: Copying image to {{ deploy_dir }}/{{ container_name }}/ + copy: + src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar' + dest: '{{ deploy_dir }}/{{ container_name }}/' + force: true + notify: + - Load Image + - Start Container + +- name: Copying {{ container_name }} config + template: + src: "admin/{{ item }}.j2" + dest: '{{ deploy_dir }}/{{ container_name }}/config/{{ item }}' + force: yes + loop: + - application.properties + - logback.xml + notify: + - Start Container + +- name: Copying {{ container_name }} docker-compose.yml + template: + src: "admin/docker-compose.yml.j2" + dest: '{{ deploy_dir }}/{{ container_name }}/docker-compose.yml' + force: yes + notify: + - Start Container + +- name: Copying {{ container_name }} nginx config + template: + src: "admin/{{ container_name }}.conf.j2" + dest: '{{ deploy_dir }}/galaxy-gateway-nginx/conf/includes/{{ container_name }}.conf' + force: yes + notify: + - Start Nginx + delegate_to: "{{ item }}" + run_once: true + loop: "{{ groups.loadbalancer[:2] }}" + +- meta: flush_handlers + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml new file mode 100644 index 0000000..7ea6d93 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml @@ -0,0 +1,45 @@ +- name: Setting variable name + set_fact: + container_name: "{{ executor_container_name }}" + image_name: "{{ executor_image_name }}" + image_tag: "{{ executor_image_tag }}" + +- name: Creating {{ container_name }} directory + file: + state: directory + path: '{{ deploy_dir }}/{{ container_name }}/{{ item }}' + loop: + - "init" + - "config" + +- name: Copying image to {{ deploy_dir }}/{{ container_name }}/ + copy: + src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar' + dest: '{{ deploy_dir }}/{{ container_name }}/' + force: true + notify: + - Load Image + - Start Container + +- name: Copying {{ container_name }} config + template: + src: "executor/{{ item }}.j2" + dest: '{{ deploy_dir }}/{{ container_name }}/config/{{ item }}' + force: yes + loop: + - application.properties + - logback.xml + notify: + - Start Container + +- name: Copying {{ container_name }} docker-compose.yml + template: + src: "executor/docker-compose.yml.j2" + dest: '{{ deploy_dir }}/{{ container_name }}/docker-compose.yml' + force: yes + notify: + - Start Container + + +- meta: flush_handlers + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml new file mode 100644 index 0000000..d3385f7 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml @@ -0,0 +1,53 @@ +- name: Creating directory + file: + state: directory + path: '{{ deploy_dir }}/{{ item }}/init' + loop: + - 'galaxy-job-admin' + - 'galaxy-job-executor' + +- name: Copying admin init files + template: + src: "{{ item.src }}" + dest: "{{ deploy_dir }}/{{ admin_container_name }}/init/{{ item.dest }}" + mode: "{{ item.mode }}" + force: true + loop: + - { src: "push_config.sh.j2", dest: "push_config.sh", mode: "0755" } + - { src: "admin/xxl_job.sql.j2", dest: "xxl_job.sql", mode: "0644" } + - { src: 'admin/create_mariadb_tables.sh.j2', dest: 'create_mariadb_tables.sh', mode: "0755" } + - { src: "admin/galaxy-job-admin.properties.j2", dest: "galaxy-job-admin.properties", mode: "0644" } + +- name: Copying mysql Client to /usr/bin/ + copy: + src: 'files/mysql' + dest: '/usr/bin/' + mode: 0755 + force: true + +- name: Creating table and init jobs + shell: cd {{ deploy_dir }}/{{ admin_container_name }}/init && sh create_mariadb_tables.sh + run_once: true + when: operation == 'install' + +- name: Copying executor init files + template: + src: "{{ item.src }}" + dest: "{{ deploy_dir }}/{{ executor_container_name }}/init/{{ item.dest }}" + mode: "{{ item.mode }}" + force: true + loop: + - { src: "push_config.sh.j2", dest: "push_config.sh", mode: "0755" } + - { src: "executor/galaxy-job-executor.properties.j2", dest: "galaxy-job-executor.properties", mode: "0644" } + +- name: Pushing Galaxy-job-service admin config + shell: 'cd {{ deploy_dir }}/{{ item.src }}/init && sh push_config.sh {{ item.conf }}' + register: post_result + until: post_result.stdout == '1' + retries: 3 + delay: 3 + failed_when: post_result.stdout != '1' + loop: + - { src: "{{ admin_container_name }}" , conf: "galaxy-job-admin.properties" } + - { src: "{{ executor_container_name }}" , conf: "galaxy-job-executor.properties" } + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml new file mode 100644 index 0000000..a1d5b98 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml @@ -0,0 +1,22 @@ +- block: + - include: uninstall-admin.yml + - include: uninstall-executor.yml + - include: init.yml + - include: deploy-admin.yml + - include: deploy-executor.yml + when: operation == 'install' + +- block: + - include: check.yml + - include: backup.yml + - include: init.yml + - include: upgrade.yml + - include: deploy-executor.yml + - include: check.yml + when: operation == 'upgrade' + +- block: + - include: uninstall-admin.yml + - include: uninstall-executor.yml + when: operation == 'uninstall' + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml new file mode 100644 index 0000000..2ebe9c8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml @@ -0,0 +1,18 @@ +- name: Stopping and removing {{ admin_container_name }} container + docker_container: + name: '{{ admin_container_name }}' + state: absent + force_kill: true + +- name: Removing old {{ admin_image_name }} image + docker_image: + name: '{{ admin_image_name }}' + tag: '{{ admin_image_tag }}' + state: absent + force_absent: true + +- name: Ansible delete old {{ deploy_dir }}/{{ admin_container_name }} + file: + path: '{{ deploy_dir }}/{{ admin_container_name }}' + state: absent + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml new file mode 100644 index 0000000..3e4deb8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml @@ -0,0 +1,18 @@ +- name: Stopping and removing {{ executor_container_name }} container + docker_container: + name: '{{ executor_container_name }}' + state: absent + force_kill: true + +- name: Removing old {{ executor_image_name }} image + docker_image: + name: '{{ executor_image_name }}' + tag: '{{ executor_image_tag }}' + state: absent + force_absent: true + +- name: Ansible delete old {{ deploy_dir }}/{{ executor_container_name }} + file: + path: '{{ deploy_dir }}/{{ executor_container_name }}' + state: absent + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml new file mode 100644 index 0000000..5da8917 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml @@ -0,0 +1,54 @@ +- name: Creating config tmp directory + file: + state: directory + path: "/tmp/{{ item }}" + loop: + - "{{ admin_container_name }}" + - "{{ executor_container_name }}" + +- name: Creating upgrade directory + file: + state: directory + path: "{{ deploy_dir }}/{{ executor_container_name }}/upgrade/upgrade-sql" + +- name: Fetching config files + fetch: + src: "{{ deploy_dir }}/{{ item.name }}/{{ item.config }}" + dest: "/tmp/{{ item.name }}/{{ item.config }}" + flat: yes + loop: + - { name: "{{ admin_container_name }}", config: 'docker-compose.yml' } + - { name: "{{ executor_container_name }}", config: 'docker-compose.yml' } + +- name: Loading variable by admin docker-compose.yml + include_vars: + file: "/tmp/{{ admin_container_name }}/docker-compose.yml" + name: admin_docker_config + +- name: Loading variable by executor docker-compose.yml + include_vars: + file: "/tmp/{{ executor_container_name }}/docker-compose.yml" + name: executor_docker_config + +- name: Set variable + set_fact: + galaxy_job_service: + admin_java_opts: "{{ admin_docker_config['services']['galaxy-job-admin']['environment']['JAVA_OPTS'] }}" + executor_java_opts: "{{ executor_docker_config['services']['galaxy-job-executor']['environment']['JAVA_OPTS'] }}" + +- name: Copying upgrade sql file + copy: + src: 'files/upgrade-sql' + dest: '{{ deploy_dir }}/{{ executor_container_name }}/upgrade/' + force: true + +- name: Copying upgrade-mariadb-data.sh template + template: + src: "upgrade-mariadb-data.sh.j2" + dest: "{{ deploy_dir }}/{{ executor_container_name }}/upgrade/upgrade-mariadb-data.sh" + mode: 0755 + force: true + +- name: Upgrade mariadb data + shell: cd {{ deploy_dir }}/{{ executor_container_name }}/upgrade && sh upgrade-mariadb-data.sh + run_once: true diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 new file mode 100644 index 0000000..6ea1881 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 @@ -0,0 +1,23 @@ +#配置集的配置格式 +nacos.config.type=properties +#配置中心地址 +nacos.config.server-addr={{ vrrp_instance.default.virtual_ipaddress }}:8848 +#命名空间 +nacos.config.namespace={{ config_namespace }} +#数据集ID +nacos.config.data-id=galaxy-job-admin.properties +#开启自动刷新 +nacos.config.auto-refresh=true +#配置对应的分组 +nacos.config.group=Galaxy +#Nacos认证用户 +nacos.config.username=nacos +#Nacos认证密码 +nacos.config.password=nacos +#开启配置预加载功能 +nacos.config.bootstrap.enable=true +#开启Nacos支持日志级别的加载时机 +nacos.config.bootstrap.log-enable=true + +## log config +logging.config=./config/logback.xml diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 new file mode 100644 index 0000000..392997f --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/bash + +mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} -e "create database if not exists {{ mariadb_job_database }} default character set utf8mb4 collate utf8mb4_general_ci;" + +mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} {{ mariadb_job_database }} < {{ deploy_dir }}/{{ admin_container_name }}/init/xxl_job.sql + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 new file mode 100644 index 0000000..be4abb8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 @@ -0,0 +1,15 @@ +version: '3' +services: + galaxy-job-admin: + image: {{ image_name }}:{{ image_tag }} + container_name: galaxy-job-admin + environment: + JAVA_OPTS: "{{ galaxy_job_service.admin_java_opts }}" + ports: + - "8181:8181" + volumes: + - "{{ deploy_dir }}/{{ container_name }}/logs:/logs" + - "{{ deploy_dir }}/{{ container_name }}/data:/data" + - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/galaxy-job/galaxy-job-admin/config" + restart: always + network_mode: "host" diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 new file mode 100644 index 0000000..bee9eea --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 @@ -0,0 +1,16 @@ +upstream jobAdmin { + +{% for host in groups['galaxy_job_service'][:2] %} + server {{ host }}:8184; +{% endfor %} + +} + +server { + listen 8181; + server_name localhost; + location / { + proxy_pass http://jobAdmin; + } +} + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 new file mode 100644 index 0000000..082e719 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 @@ -0,0 +1,73 @@ +### web +server.port=8184 +server.servlet.context-path=/xxl-job-admin +spring.application.name=galaxy-job-admin +### actuator +management.server.servlet.context-path=/actuator +management.health.mail.enabled=false +management.endpoints.web.exposure.include=* +#详细的应用健康信息 prometheus +management.endpoint.health.show-details=always + +management.endpoint.metrics.enabled=true +management.endpoint.prometheus.enabled=true +management.metrics.export.prometheus.enabled=true +management.metrics.tags.application=${spring.application.name} +management.metrics.tags.module=${spring.application.name} + + +### resources +spring.mvc.servlet.load-on-startup=0 +spring.mvc.static-path-pattern=/static/** +spring.resources.static-locations=classpath:/static/ + +### freemarker +spring.freemarker.templateLoaderPath=classpath:/templates/ +spring.freemarker.suffix=.ftl +spring.freemarker.charset=UTF-8 +spring.freemarker.request-context-attribute=request +spring.freemarker.settings.number_format=0.########## + +### mybatis +mybatis.mapper-locations=classpath:/mybatis-mapper/*Mapper.xml +#mybatis.type-aliases-package=com.xxl.job.admin.core.model + +### xxl-job, datasource +spring.datasource.url=jdbc:mysql://{{ vrrp_instance.default.virtual_ipaddress }}:3306/{{ mariadb_job_database }}?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&serverTimezone=UTC +spring.datasource.username=ENC(63aTpwv2vH0vPikW+3Jjig==) +spring.datasource.password=ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y) +spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver + +### datasource-pool +spring.datasource.type=com.zaxxer.hikari.HikariDataSource +spring.datasource.hikari.minimum-idle=10 +spring.datasource.hikari.maximum-pool-size=100 +spring.datasource.hikari.auto-commit=true +spring.datasource.hikari.idle-timeout=30000 +spring.datasource.hikari.pool-name=HikariCP +spring.datasource.hikari.max-lifetime=900000 +spring.datasource.hikari.connection-timeout=30000 +spring.datasource.hikari.connection-test-query=SELECT 1 + +### xxl-job, email +spring.mail.host=smtp.qq.com +spring.mail.port=25 +spring.mail.username=xxx@qq.com +spring.mail.password=xxx +spring.mail.properties.mail.smtp.auth=true +spring.mail.properties.mail.smtp.starttls.enable=true +spring.mail.properties.mail.smtp.starttls.required=true +spring.mail.properties.mail.smtp.socketFactory.class=javax.net.ssl.SSLSocketFactory + +### xxl-job, access token +xxl.job.accessToken= + +### xxl-job, i18n (default is zh_CN, and you can choose "zh_CN", "zh_TC" and "en") +xxl.job.i18n=zh_CN + +## xxl-job, triggerpool max size +xxl.job.triggerpool.fast.max=200 +xxl.job.triggerpool.slow.max=100 + +### xxl-job, log retention days +xxl.job.logretentiondays=30 diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 new file mode 100644 index 0000000..dd729b0 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + ${LOG_PATTERN} + + + + + ${LOG_PATH}/${LOG_FILE_NAME}.log + + ALL + + + ${LOG_PATTERN} + + + + ${LOG_PATH}/history/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz + + ${LOG_DAYS} + ${LOG_SIZE} + + + + + + + + + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 new file mode 100644 index 0000000..f3562c8 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 @@ -0,0 +1,195 @@ +/* +Navicat MySQL Data Transfer + +Target Server Type : MYSQL +Target Server Version : 50505 +File Encoding : 65001 + +Date: 2021-09-23 14:05:11 +*/ + +SET FOREIGN_KEY_CHECKS=0; + +-- ---------------------------- +-- Table structure for `xxl_job_group` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_group`; +CREATE TABLE `xxl_job_group` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `app_name` varchar(64) NOT NULL COMMENT '执行器AppName', + `title` varchar(64) NOT NULL COMMENT '执行器名称', + `address_type` tinyint(4) NOT NULL DEFAULT 0 COMMENT '执行器地址类型：0=自动注册、1=手动录入', + `address_list` varchar(512) DEFAULT NULL COMMENT '执行器地址列表，多地址逗号分隔', + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Table structure for `xxl_job_lock` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_lock`; +CREATE TABLE `xxl_job_lock` ( + `lock_name` varchar(50) NOT NULL COMMENT '锁名称', + PRIMARY KEY (`lock_name`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Records of xxl_job_lock +-- ---------------------------- +INSERT INTO `xxl_job_lock` VALUES ('schedule_lock'); + +-- ---------------------------- +-- Table structure for `xxl_job_log` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_log`; +CREATE TABLE `xxl_job_log` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_group` int(11) NOT NULL COMMENT '执行器主键ID', + `job_id` int(11) NOT NULL COMMENT '任务，主键ID', + `executor_address` varchar(255) DEFAULT NULL COMMENT '执行器地址，本次执行的地址', + `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', + `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数', + `executor_sharding_param` varchar(20) DEFAULT NULL COMMENT '执行器任务分片参数，格式如 1/2', + `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数', + `trigger_time` datetime DEFAULT NULL COMMENT '调度-时间', + `trigger_code` int(11) NOT NULL COMMENT '调度-结果', + `trigger_msg` text DEFAULT NULL COMMENT '调度-日志', + `handle_time` datetime DEFAULT NULL COMMENT '执行-时间', + `handle_code` int(11) NOT NULL COMMENT '执行-状态', + `handle_msg` text DEFAULT NULL COMMENT '执行-日志', + `alarm_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '告警状态：0-默认、1-无需告警、2-告警成功、3-告警失败', + PRIMARY KEY (`id`) USING BTREE, + KEY `I_trigger_time` (`trigger_time`) USING BTREE, + KEY `I_handle_code` (`handle_code`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=2077229 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Records of xxl_job_log +-- ---------------------------- + +-- ---------------------------- +-- Table structure for `xxl_job_logglue` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_logglue`; +CREATE TABLE `xxl_job_logglue` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `job_id` int(11) NOT NULL COMMENT '任务，主键ID', + `glue_type` varchar(50) DEFAULT NULL COMMENT 'GLUE类型', + `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码', + `glue_remark` varchar(128) NOT NULL COMMENT 'GLUE备注', + `add_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Records of xxl_job_logglue +-- ---------------------------- + +-- ---------------------------- +-- Table structure for `xxl_job_log_report` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_log_report`; +CREATE TABLE `xxl_job_log_report` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `trigger_day` datetime DEFAULT NULL COMMENT '调度-时间', + `running_count` int(11) NOT NULL DEFAULT 0 COMMENT '运行中-日志数量', + `suc_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行成功-日志数量', + `fail_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行失败-日志数量', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `i_trigger_day` (`trigger_day`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=786 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Records of xxl_job_log_report +-- ---------------------------- + +-- ---------------------------- +-- Table structure for `xxl_job_registry` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_registry`; +CREATE TABLE `xxl_job_registry` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `registry_group` varchar(50) NOT NULL, + `registry_key` varchar(255) NOT NULL, + `registry_value` varchar(255) NOT NULL, + `update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE, + KEY `i_g_k_v` (`registry_group`,`registry_key`,`registry_value`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=276 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Table structure for `xxl_job_user` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_user`; +CREATE TABLE `xxl_job_user` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `username` varchar(50) NOT NULL COMMENT '账号', + `password` varchar(50) NOT NULL COMMENT '密码', + `role` tinyint(4) NOT NULL COMMENT '角色：0-普通用户、1-管理员', + `permission` varchar(255) DEFAULT NULL COMMENT '权限：执行器ID列表，多个逗号分割', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `i_username` (`username`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + +-- ---------------------------- +-- Records of xxl_job_user +-- ---------------------------- +INSERT INTO `xxl_job_user` VALUES ('1', 'admin', 'fea191a3fdd9f68503f9fa0e8d0293ab', '1', null); +INSERT INTO `xxl_job_user` VALUES ('2', 'query', '95497ea23bf8d27d86526717578e366f', '0', '3'); + +-- ---------------------------- +-- Table structure for `xxl_job_info` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_info`; +CREATE TABLE `xxl_job_info` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `job_group` int(11) NOT NULL COMMENT '执行器主键ID', + `job_cron` varchar(128) NOT NULL COMMENT '任务执行CRON', + `job_desc` varchar(255) NOT NULL, + `add_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `author` varchar(64) DEFAULT NULL COMMENT '作者', + `alarm_email` varchar(255) DEFAULT NULL COMMENT '报警邮件', + `executor_route_strategy` varchar(50) DEFAULT NULL COMMENT '执行器路由策略', + `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', + `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数', + `executor_block_strategy` varchar(50) DEFAULT NULL COMMENT '阻塞处理策略', + `executor_timeout` int(11) NOT NULL DEFAULT 0 COMMENT '任务执行超时时间，单位秒', + `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数', + `glue_type` varchar(50) NOT NULL COMMENT 'GLUE类型', + `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码', + `glue_remark` varchar(128) DEFAULT NULL COMMENT 'GLUE备注', + `glue_updatetime` datetime DEFAULT NULL COMMENT 'GLUE更新时间', + `child_jobid` varchar(255) DEFAULT NULL COMMENT '子任务ID，多个逗号分隔', + `trigger_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '调度状态：0-停止，1-运行', + `trigger_last_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '上次调度时间', + `trigger_next_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '下次调度时间', + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=88 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC; + + +-- ---------------------------- +-- Records of xxl_job_group +-- ---------------------------- +INSERT INTO `xxl_job_group` VALUES (1, 'galaxy-executor', 'GDP', 1, '{{ job_executor_servers }}'); +INSERT INTO `xxl_job_group` VALUES (2, 'galaxy-executor', 'TSG_OLAP', 1, '{{ job_executor_servers }}'); +-- ---------------------------- +-- Records of xxl_job_info +-- ---------------------------- +INSERT INTO `xxl_job_info` VALUES (58, 1, '0 0 0 1/1 * ? ', 'set_druid_table_ttl', '2020-07-30 10:34:34', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteReportAndMetricsDataJobHandler', '{\"defaultMaxDays\":365,\"maxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 10:34:34', '', 1, 1704326400000, 1704412800000); +INSERT INTO `xxl_job_info` VALUES (59, 1, '0 0 0 1 * ?', 'clear_druid_table', '2020-07-30 11:32:08', '2022-04-12 02:38:39', 'zhq', '', 'FAILOVER', 'deleteAllReportAndMetricsDataJobHandler', '{\"maxDays\":88,\"defaultMaxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:32:08', '', 0, 0, 0); +INSERT INTO `xxl_job_info` VALUES (62, 1, '0 0 0 1/1 * ? ', 'set_hos_bucket_ttl', '2020-07-30 11:59:30', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteFilesJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:30', '', 1, 1704326400000, 1704412800000); +INSERT INTO `xxl_job_info` VALUES (63, 1, '0 0 0 1 * ?', 'clear_hos_file', '2020-07-30 11:59:43', '2023-10-26 10:43:46', 'zhq', '', 'FAILOVER', 'deleteAllFilesJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:43', '', 0, 0, 0); +INSERT INTO `xxl_job_info` VALUES (64, 1, '0 0 0 1 * ?', 'clear_clickhouse_table', '2020-07-30 13:47:25', '2022-04-12 02:38:39', 'zhq', '', 'FAILOVER', 'deleteAllTrafficDataJobHandler', '{\"maxDays\":77,\"defaultMaxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:25', '', 0, 0, 0); +INSERT INTO `xxl_job_info` VALUES (65, 1, '0 0 0 1/1 * ? ', 'set_clickhouse_table_ttl', '2020-07-30 13:47:38', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteTrafficDataJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:38', '', 1, 1704326400000, 1704412800000); + +INSERT INTO `xxl_job_info` VALUES (29, 2, '0 0 4 * * ?', 'get_clickhoue_storage_daily_usage', '2020-03-26 10:58:09', '2021-09-09 11:54:53', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE\",\"ckDayGrowth\":\"true\"}', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-03-26 10:58:09', '', 1, 1704254400000, 1704340800000); +INSERT INTO `xxl_job_info` VALUES (36, 2, '0 0/5 * * * ?', 'get_database_storage_quota', '2020-04-07 21:41:20', '2021-09-09 11:54:44', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE\",\"ckDayGrowth\":\"false\"}', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-04-07 21:41:20', '', 1, 1704338100000, 1704338400000); +INSERT INTO `xxl_job_info` VALUES (86, 2, '0 0 1 * * ?', 'run_druid_segment_compaction', '2021-07-21 14:13:57', '2023-12-12 02:21:05', 'xpf', '', 'FAILOVER', 'druidCompactSegmentJobHandler', '[\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_server_fqdns\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_server_domains\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_client_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_server_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_external_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_internal_ips\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"top_subscribers\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"security_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"monitor_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"proxy_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"application_protocol_stat\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"traffic_general_stat\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"traffic_top_destination_ip_metrics_log\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"service_chaining_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"service_function_status\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"statistics_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"traffic_shaping_rule_hits\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"object_statistics\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"statistics_rule\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\",\r\n \"maxRowsInMemory\": 500000,\r\n \"maxBytesInMemory\": -1\r\n },\r\n \"context\":{\r\n \"druid.indexer.runner.javaOptsArray\": [\"-Xms4g\", \"-Xmx4g\", \"-XX:MaxDirectMemorySize=2g\"]\r\n }\r\n },\r\n {\r\n \"type\": \"compact\",\r\n \"dataSource\": \"sys_storage_log\",\r\n \"tuningConfig\": {\r\n \"type\": \"index_parallel\"\r\n }\r\n }\r\n]', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2021-07-21 14:13:57', '', 1, 1704330000000, 1704416400000); +INSERT INTO `xxl_job_info` VALUES (101, 2, '0 0 0 1/7 * ?', 'run_web_sketch_assessment', '2022-01-27 17:40:42', '2022-02-23 13:34:51', 'galaxy', '', 'FAILOVER', 'httpToStoreJobHandler', '[{\r\n \"url\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/sql/?option=long_term&resultId=1&query=SELECT server_domain AS name, arrayStringConcat(groupUniqArray(decoded_as),\',\') AS protocol, count(*) AS sessions FROM tsg_galaxy_v3.session_record WHERE recv_time >= toStartOfDay(now()- INTERVAL 7 DAY) AND recv_time < toStartOfDay(now()) AND notEmpty(server_domain) GROUP BY name ORDER BY sessions DESC LIMIT 500000;\",\r\n \"method\": \"get\",\r\n \"requestBody\": {},\r\n \"resultKey\": \"data\",\r\n \"store\": \"all\"\r\n}, {\r\n \"url\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/sql/?option=long_term&resultId=1&query=SELECT server_domain AS name, arrayStringConcat(groupUniqArray(decoded_as),\',\') AS protocol, count(*) AS sessions FROM tsg_galaxy_v3.session_record WHERE recv_time >= toStartOfDay(now()- INTERVAL 7 DAY) AND recv_time < toStartOfDay(now()) AND notEmpty(server_domain) AND empty(fqdn_category_list) GROUP BY name ORDER BY sessions DESC LIMIT 500000;\",\r\n \"method\": \"get\",\r\n \"requestBody\": {},\r\n \"resultKey\": \"data\",\r\n \"store\": \"uncategorized\"\r\n}]', 'COVER_EARLY', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2022-01-27 17:40:42', '', 1, 1704067200000, 1704672000000); +INSERT INTO `xxl_job_info` VALUES (141, 2, '0 0 2 * * ?', 'set_clickhouse_table_and_field_ttl', '2022-06-07 07:42:38', '2022-06-08 09:28:19', 'wk', '', 'FIRST', 'changeCkTtlJobHandler', '', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2022-06-07 07:42:38', '', 1, 1704333600000, 1704420000000); +INSERT INTO `xxl_job_info` VALUES (143, 2, '0 0 0/1 * * ?', 'run_ip_learning', '2022-11-09 10:29:20', '2023-12-26 04:01:24', 'galaxy', '', 'FIRST', '', '', 'SERIAL_EXECUTION', 0, 0, 'GLUE_SHELL', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/iplearning/ && ./start.sh\'', '0001', '2023-12-26 04:01:24', '', 1, 1704337200000, 1704340800000); +INSERT INTO `xxl_job_info` VALUES (145, 2, '0 0 1 * * ?', 'run_dos_baseline', '2022-11-10 06:15:27', '2023-11-13 05:37:41', 'galaxy', '', 'FIRST', '', '', 'SERIAL_EXECUTION', 0, 0, 'GLUE_SHELL', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/dos-baseline && ./start.sh\'', '0000', '2023-11-13 05:37:41', '', 1, 1704330000000, 1704416400000); +INSERT INTO `xxl_job_info` VALUES (155, 2, '0 0 16 1/7 * ?', 'run_geoip_to_mmdb', '2022-12-02 06:56:18', '2023-12-21 09:44:44', 'galaxy', '', 'FAILOVER', 'ipLocateLibraryJobHandler', '{\r\n \"command\": \"docker run --rm --network=host -v {{ deploy_dir }}/schedule-service/mmdb:/mmdb --name perl-mmdb perl:v3.1 sh -c \'cd /mmdb && perl mmdb-perl.pl\' \",\r\n \"path\": \"/opt/schedule-service/mmdb\",\r\n \"knowledgeBaseServer\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/v1/knowledge_base\",\r\n \"knowledgeBaseList\": [{\"name\": \"ip_builtin\",\"builtin\":1 ,\"kb_id\": \"64af7077-eb9b-4b8f-80cf-2ceebc89bea9\" },\r\n {\"name\": \"ip_user_defined\",\"builtin\":0 ,\"kb_id\": \"004390bc-3135-4a6f-a492-3662ecb9e289\" }]\r\n}', 'DISCARD_LATER', 0, 0, 'BEAN', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/dos-baseline && ./start.sh\'', 'GLUE代码初始化', '2022-12-02 06:56:18', '', 0, 0, 0); +INSERT INTO `xxl_job_info` VALUES (158, 2, '0 0 1 * * ?', 'run_druid_segement_granularity_compact', '2023-02-20 05:59:59', '2023-05-11 03:39:14', 'galaxy', '', 'FAILOVER', 'druidMergeHistoryDataJobHandler', '[\r\n {\r\n \"dataSource\": \"service_function_status\",\r\n \"internalOrNumOfHistory\": \"1\",\r\n \"queryGranularity\": \"PT5M\",\r\n \"segmentGranularity\": \"DAY\"\r\n },\r\n {\r\n \"dataSource\": \"service_chaining_rule_hits\",\r\n \"internalOrNumOfHistory\": \"1\",\r\n \"queryGranularity\": \"PT5M\",\r\n \"segmentGranularity\": \"DAY\"\r\n },\r\n {\r\n \"dataSource\": \"traffic_shaping_rule_hits\",\r\n \"internalOrNumOfHistory\": \"1\",\r\n \"queryGranularity\": \"PT5M\",\r\n \"segmentGranularity\": \"DAY\"\r\n }\r\n]', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2023-02-20 05:59:59', '', 1, 1704330000000, 1704416400000); + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 new file mode 100644 index 0000000..205a6a0 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 @@ -0,0 +1,23 @@ +#配置集的配置格式 +nacos.config.type=properties +#配置中心地址 +nacos.config.server-addr={{ vrrp_instance.default.virtual_ipaddress }}:8848 +#命名空间 +nacos.config.namespace={{ config_namespace }} +#数据集ID +nacos.config.data-id=galaxy-job-executor.properties +#开启自动刷新 +nacos.config.auto-refresh=true +#配置对应的分组 +nacos.config.group=Galaxy +#Nacos认证用户 +nacos.config.username=nacos +#Nacos认证密码 +nacos.config.password=nacos +#开启配置预加载功能 +nacos.config.bootstrap.enable=true +#开启Nacos支持日志级别的加载时机 +nacos.config.bootstrap.log-enable=true + +### log config +logging.config=./config/logback.xml diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 new file mode 100644 index 0000000..1aab54c --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 @@ -0,0 +1,19 @@ +version: '3' +services: + galaxy-job-executor: + image: {{ image_name }}:{{ image_tag }} + container_name: galaxy-job-executor + environment: + JAVA_OPTS: "{{ galaxy_job_service.executor_java_opts }}" + ports: + - "8182:8182" + volumes: + - "{{ deploy_dir }}/{{ container_name }}/logs:/logs" + - "{{ deploy_dir }}/{{ container_name }}/data:/data" + - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/galaxy-job/galaxy-job-executor/config" + - "{{ deploy_dir }}/schedule-service:/opt/schedule-service" + - "/var/run/docker.sock:/var/run/docker.sock" + - "/usr/bin/docker:/usr/bin/docker" + restart: always + network_mode: "host" + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 new file mode 100644 index 0000000..5c8b0ef --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 @@ -0,0 +1,65 @@ +################################静态参数配置(修改后需要重启项目)################################ +### web port +server.port = 8185 +spring.application.name=galaxy-job-executor +### actuator +management.server.servlet.context-path=/actuator +management.health.mail.enabled=false +management.endpoints.web.exposure.include=* +#详细的应用健康信息 +management.endpoint.health.show-details=always +management.endpoint.metrics.enabled=true +management.endpoint.prometheus.enabled=true +management.metrics.export.prometheus.enabled=true +management.metrics.tags.application=${spring.application.name} +zookeeper.server={{ zookeeper_servers }} + +################################动态参数配置(修改后不需要重启项目)################################ +##存储配额文件服务器 +storge.files.hos-server={{ data_center_name }}|{{ vrrp_instance.oss.virtual_ipaddress }}:9098 +storge.files.token={{ hos_token }} +##存储配额查询druid +storge.analytic.server={{ data_center_name }}|{{ vrrp_instance.default.virtual_ipaddress }}:8089 +##存储配额查询clickhouse +storge.traffic.server={{ data_center_name }}|{{ vrrp_instance.default.virtual_ipaddress }}:8124 +storge.traffic.datasource=tsg_galaxy_v3 +storge.traffic.username=ENC(CNpgJlarVTNItkyW6DAQWA==) +storge.traffic.password=ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y) +#删除ttl +storge.traffic.system.parts=system.parts +#存储配额查询 +storge.traffic.system.partsclusters=system.parts_cluster +storge.traffic.system.disks=system.disks_cluster +storge.traffic.system.tables=system.tables_cluster +storge.traffic.system.clusters=system.clusters +#删除ttl白名单,多个逗号分隔 +storge.files.delete.exclusion=default,assessment_hos_bucket,knowledge_base_hos_bucket +storge.analytic.delete.exclusion=traffic_general_stat +storge.taffic.delete.exclusion=assessment_event_local +### xxl-job admin address list, such as "http://address" or "http://address01,http://address02" +xxl.job.admin.addresses={{ job_admin_servers }} +### xxl-job, access token +xxl.job.accessToken= +### xxl-job executor registry-address: default use address to registry , otherwise use ip:po +xxl.job.executor.appname=galaxy-executor +### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null +xxl.job.executor.address= +### xxl-job executor server-info +xxl.job.executor.ip= +xxl.job.executor.port=8886 +### xxl-job executor log-path +xxl.job.executor.logpath=/data/logs/jobhandler +### xxl-job executor log-retention-days +xxl.job.executor.logretentiondays=30 +## http pool config +### max connection number +http.pool.max.connection=500 +http.pool.request.timeout=120000 +http.pool.response.timeout=300000 +http.pool.max.per.route=300 +http.pool.connect.timeout=10000 +##指定kafka server的地址，集群配多个，中间，逗号隔开 +spring.kafka.bootstrap-servers={{ kafka_sink_servers }} +spring.kafka.ssl.enable=true +spring.kafka.ssl.username=ENC(RYN0it9o1vYynI1xlmX/Gw==) +spring.kafka.ssl.pin=ENC(vlb4F5Y3ff0Am/3MMPdIpYjZq8wcGNMd) diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 new file mode 100644 index 0000000..758d04d --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + ${LOG_PATTERN} + + + + ${LOG_PATH}/${LOG_FILE_NAME}.log + + ALL + + + ${LOG_PATTERN} + + + + ${LOG_PATH}/history/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz + + ${LOG_DAYS} + ${LOG_SIZE} + + + + + + + + + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 new file mode 100644 index 0000000..87689e5 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 @@ -0,0 +1,52 @@ +#!/bin/bash + +config_path_root=$1 +nacos_server={{ vrrp_instance.default.virtual_ipaddress }} + +tenant="{{ config_namespace }}" +app_name="galaxy-job-service" + +shell_result=1 + +# 推送配置 +function push() { + local config_file=$1 + local data_id="${config_file##*/}" + local suffix="${config_file##*.}" + + [ $suffix == "sql" ] && suffix="text" + [ $suffix == "yml" ] && suffix="yaml" + + local nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy" + local result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content@"$config_file") + if [[ $result -eq '200' ]]; then + sleep 2 + check=$(curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id") + #echo "$check $data_id" + + if [[ $check -ne '200' ]]; then + shell_result=0 + fi + fi +} + +# 获取本地文件 +function pushConfig() { + for config_path in $(readlink -f $config_path_root); do + if [ -d $config_path ]; then + for file in $(ls $config_path); do + config_path_file=$config_path/$file + if [ -f $config_path_file ]; then + push $config_path_file + fi + done + fi + if [ -f $config_path ]; then + push $config_path + fi + done +} + +pushConfig +echo $shell_result + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 new file mode 100644 index 0000000..bfd7c17 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 @@ -0,0 +1,50 @@ +#!/bin/bash +set -e + +# 数据库连接参数 +upgrade_dir=${1:-`pwd`/upgrade-sql} + +# 指定当前版本和历史版本 +current_version="{{ solution_version }}" +old_version="{{ old_version }}" + +# 数据库连接参数 +db_hostname="{{ vrrp_instance.default.virtual_ipaddress }}" +db_name="xxl_job" +db_username="root" +db_password="{{ mariadb_default_pin }}" + +suffix=".sql" +# 获取更新目录中的所有SQL文件 +sql_files=$(find "${upgrade_dir}" -name "*$suffix") + +# 标记是否开始执行历史版本的标识 +start_execution=false + +# 循环处理每个SQL文件 +for file in ${sql_files}; do + # 从文件名中提取版本号 + filename=$(basename "$file") + version=$(echo "$filename" | grep -oE '[0-9]+(\.[0-9]+)?' | tr -d '.') + current_version="${current_version//./}" + old_version="${old_version//./}" + # 检查是否达到历史版本 + if [[ "${version}" -gt "${old_version}" ]]; then + start_execution=true + fi + + # 检查是否开始执行历史版本之后的SQL文件 + if [[ ${start_execution} = true ]]; then + echo "Executing SQL file: ${file}" + + # 执行SQL文件 + mysql -u "${db_username}" -p"${db_password}" -h "${db_hostname}" "${db_name}" < "${file}" + echo "Upgrade ${version} successfully" + fi + + # 检查是否达到当前版本 + if [[ "${version}" -ge "${current_version}" ]]; then + break + fi +done + diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml new file mode 100644 index 0000000..de010b3 --- /dev/null +++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml @@ -0,0 +1,26 @@ +software_packages_dir: "{{ role_path }}/../software-packages" + +admin_image_name: galaxy-job-admin +executor_image_name: galaxy-job-executor + +admin_image_tag: v1.3.220308 +executor_image_tag: v1.3.231220 + +admin_container_name: galaxy-job-admin +executor_container_name: galaxy-job-executor + + +mariadb_job_database: xxl_job +job_executor_servers: "{% for host in groups['galaxy_job_service'] %}http://{{ host }}:8886/{% if not loop.last %},{% endif %}{% endfor %}" + +job_admin_servers: "{% for host in groups['galaxy_job_service'] %}http://{{ host }}:8184/xxl-job-admin{% if not loop.last %},{% endif %}{% endfor %}" + +kafka_sink_servers: "{% for host in groups['kafka'] %}{{ host }}:9094{% if not loop.last %},{% endif %}{% endfor %}" +zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ host }}:2181{% if not loop.last %},{% endif %}{% endfor %}" + +backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/" + +backup_items: + - config + - docker-compose.yml + - init