diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..6504beb --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,33 @@ +image: 192.168.40.153:8082/common/maven:3.8.1-openjdk-11-slim + +stages: + - build + - deploy + + +variables: + VERSION: "$CI_COMMIT_TAG" + +build: + stage: build + script: + - echo "Building the project..." + - tar -czvf tsg-olap-data-initialization-$VERSION.tar.gz --exclude='*.log' --exclude='.gitlab-ci.yml' --exclude='.idea' + - ls -lah + artifacts: + paths: + - tsg-olap-data-initialization-$CI_COMMIT_TAG.tar.gz + only: + - tags + + +deploy: + stage: deploy + script: + - echo "Uploading to Nexus..." + - ls -lah + - curl -v -u $NEXUS_USER:$NEXUS_PASSWORD --upload-file tsg-olap-data-initialization-$VERSION.tar.gz "$NEXUS_URL/deployment/tsg-olap-data-initialization-$VERSION.tar.gz" + dependencies: + - build + only: + - tags \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a0b887e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ + +# Changelog + +## [24.10] - 2024-11-07 + +### Added + +### Fixed + +### Deleted + +## [24.09] - 2024-11-07 + +### Initial release +- 发布TSG OLAP 安装包所有的配置文件和初始化脚本。 \ No newline at end of file diff --git a/README.md b/README.md index 7f34fef..a6800ff 100644 --- a/README.md +++ b/README.md @@ -1,93 +1,8 @@ # TSG OLAP Data Initialization +## Directory Overview - -## Getting started - -To make it easy for you to get started with GitLab, here's a list of recommended next steps. - -Already a pro? Just edit this README.md and make it your own. Want to make it easy? [Use the template at the bottom](#editing-this-readme)! - -## Add your files - -- [ ] [Create](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#create-a-file) or [upload](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#upload-a-file) files -- [ ] [Add files using the command line](https://docs.gitlab.com/ee/gitlab-basics/add-file.html#add-a-file-using-the-command-line) or push an existing Git repository with the following command: - -``` -cd existing_repo -git remote add origin https://git.mesalab.cn/galaxy/deployment/tsg-olap-data-initialization.git -git branch -M main -git push -uf origin main -``` - -## Integrate with your tools - -- [ ] [Set up project integrations](https://git.mesalab.cn/galaxy/deployment/tsg-olap-data-initialization/-/settings/integrations) - -## Collaborate with your team - -- [ ] [Invite team members and collaborators](https://docs.gitlab.com/ee/user/project/members/) -- [ ] [Create a new merge request](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html) -- [ ] [Automatically close issues from merge requests](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#closing-issues-automatically) -- [ ] [Enable merge request approvals](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/) -- [ ] [Set auto-merge](https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html) - -## Test and Deploy - -Use the built-in continuous integration in GitLab. - -- [ ] [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/quick_start/index.html) -- [ ] [Analyze your code for known vulnerabilities with Static Application Security Testing (SAST)](https://docs.gitlab.com/ee/user/application_security/sast/) -- [ ] [Deploy to Kubernetes, Amazon EC2, or Amazon ECS using Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/requirements.html) -- [ ] [Use pull-based deployments for improved Kubernetes management](https://docs.gitlab.com/ee/user/clusters/agent/) -- [ ] [Set up protected environments](https://docs.gitlab.com/ee/ci/environments/protected_environments.html) - -*** - -# Editing this README - -When you're ready to make this README your own, just edit this file and use the handy template below (or feel free to structure it however you want - this is just a starting point!). Thanks to [makeareadme.com](https://www.makeareadme.com/) for this template. - -## Suggestions for a good README - -Every project is different, so consider which of these sections apply to yours. The sections used in the template are suggestions for most open source projects. Also keep in mind that while a README can be too long and detailed, too long is better than too short. If you think your README is too long, consider utilizing another form of documentation rather than cutting out information. - -## Name -Choose a self-explaining name for your project. - -## Description -Let people know what your project can do specifically. Provide context and add a link to any reference visitors might be unfamiliar with. A list of Features or a Background subsection can also be added here. If there are alternatives to your project, this is a good place to list differentiating factors. - -## Badges -On some READMEs, you may see small images that convey metadata, such as whether or not all the tests are passing for the project. You can use Shields to add some to your README. Many services also have instructions for adding a badge. - -## Visuals -Depending on what you are making, it can be a good idea to include screenshots or even a video (you'll frequently see GIFs rather than actual videos). Tools like ttygif can help, but check out Asciinema for a more sophisticated method. - -## Installation -Within a particular ecosystem, there may be a common way of installing things, such as using Yarn, NuGet, or Homebrew. However, consider the possibility that whoever is reading your README is a novice and would like more guidance. Listing specific steps helps remove ambiguity and gets people to using your project as quickly as possible. If it only runs in a specific context like a particular programming language version or operating system or has dependencies that have to be installed manually, also add a Requirements subsection. - -## Usage -Use examples liberally, and show the expected output if you can. It's helpful to have inline the smallest example of usage that you can demonstrate, while providing links to more sophisticated examples if they are too long to reasonably include in the README. - -## Support -Tell people where they can go to for help. It can be any combination of an issue tracker, a chat room, an email address, etc. - -## Roadmap -If you have ideas for releases in the future, it is a good idea to list them in the README. - -## Contributing -State if you are open to contributions and what your requirements are for accepting them. - -For people who want to make changes to your project, it's helpful to have some documentation on how to get started. Perhaps there is a script that they should run or some environment variables that they need to set. Make these steps explicit. These instructions could also be useful to your future self. - -You can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser. - -## Authors and acknowledgment -Show your appreciation to those who have contributed to the project. - -## License -For open source projects, say how it is licensed. - -## Project status -If you have run out of energy or time for your project, put a note at the top of the README saying that development has slowed down or stopped completely. Someone may choose to fork your project or volunteer to step in as a maintainer or owner, allowing your project to keep going. You can also make an explicit request for maintainers. +| Directory Name | Description | +|:-----------------|:--------------| +| shell-scripts | 存储安装和初始化脚本。 | +| config-templates | 存储配置文件模板。 | diff --git a/clickhouse/tsg_olap_clickhouse_ddl.sql b/clickhouse/tsg_olap_clickhouse_ddl.sql new file mode 100644 index 0000000..0840c6c --- /dev/null +++ b/clickhouse/tsg_olap_clickhouse_ddl.sql @@ -0,0 +1,3391 @@ +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster; + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + sessions Int64, + session_rate Int64, + packets Int64, + packet_rate Int64, + bytes Int64, + bit_rate Int64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,destination_ip,recv_time,log_id); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster ( + vsys_id Int32, + recv_time Int64, + log_id UInt64, + profile_id Int64, + rule_id Int64, + start_time Int64, + end_time Int64, + attack_type String, + severity String, + conditions String, + destination_ip String, + destination_country String, + source_ip_list String, + source_country_list String, + sessions Int64, + session_rate Int64, + packets Int64, + packet_rate Int64, + bytes Int64, + bit_rate Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,recv_time,log_id); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + assessment_date Int64, + lot_number String, + file_name String, + assessment_file String, + assessment_type String, + features String, + size Int64, + file_checksum_sha String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand()); + + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +direction String, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_ip_tags Array(String), +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_ip_tags Array(String), +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_fqdn_tags Array(String), +server_domain String, +app_transition String,  +app LowCardinality(String), +app_category String, +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +sip_bye_reason String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +direction String, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_ip_tags Array(String), +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_ip_tags Array(String), +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_fqdn_tags Array(String), +server_domain String, +app_transition String,  +app LowCardinality(String), +app_category String, +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +sip_bye_reason String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +direction String, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_ip_tags Array(String), +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_ip_tags Array(String), +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_fqdn_tags Array(String), +server_domain String, +app_transition String,  +app LowCardinality(String), +app_category String, +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +sip_bye_reason String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +direction String, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_ip_tags Array(String), +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_ip_tags Array(String), +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_fqdn_tags Array(String), +server_domain String, +app_transition String,  +app LowCardinality(String), +app_category String, +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +sip_bye_reason String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +direction String, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_ip_tags Array(String), +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_ip_tags Array(String), +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_fqdn_tags Array(String), +server_domain String, +app_transition String,  +app LowCardinality(String), +app_category String, +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +sip_bye_reason String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +direction String, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_ip_tags Array(String), +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_ip_tags Array(String), +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_fqdn_tags Array(String), +server_domain String, +app_transition String,  +app LowCardinality(String), +app_category String, +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +ssl_version String, +ssl_sni String, +ssl_san String, +ssl_cn String, +ssl_handshake_latency_ms Nullable(Int32), +ssl_ja3_hash String, +ssl_ja3s_hash String, +ssl_cert_issuer String, +ssl_cert_subject String, +ssl_esni_flag Nullable(Int32), +ssl_ech_flag Nullable(Int32), +dtls_cookie String, +dtls_version  String, +dtls_sni String, +dtls_san String, +dtls_cn String, +dtls_handshake_latency_ms Nullable(Int32), +dtls_ja3_fingerprint String, +dtls_ja3_hash String, +dtls_cert_issuer String, +dtls_cert_subject String, +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +ftp_account String, +ftp_url String, +ftp_link_type String, +quic_version String, +quic_sni String, +quic_user_agent String, +rdp_cookie String, +rdp_security_protocol String, +rdp_client_channels String, +rdp_keyboard_layout String, +rdp_client_version String, +rdp_client_name String, +rdp_client_product_id String, +rdp_desktop_width String, +rdp_desktop_height String, +rdp_requested_color_depth String, +rdp_certificate_type String, +rdp_certificate_count Nullable(Int32), +rdp_certificate_permanent Nullable(Int32), +rdp_encryption_level String, +rdp_encryption_method String, +ssh_version String, +ssh_auth_success String, +ssh_client_version String, +ssh_server_version String, +ssh_cipher_alg String, +ssh_mac_alg String, +ssh_compression_alg String, +ssh_kex_alg String, +ssh_host_key_alg String, +ssh_host_key String, +ssh_hassh String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +sip_bye_reason String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +stratum_cryptocurrency String, +stratum_mining_pools String, +stratum_mining_program String, +stratum_mining_subscribe String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +sip_bye_reason String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,session_id,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +ingestion_time Int64, +processing_time Int64, +insert_time Int64 , +address_type Int32, +vsys_id Int32, +client_ip String, +client_port Int32, +server_ip String, +server_port Int32, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +dns_message_id Nullable(Int32), +dns_qr Nullable(Int32), +dns_opcode Nullable(Int32), +dns_aa Nullable(Int32), +dns_tc Nullable(Int32), +dns_rd Nullable(Int32), +dns_ra Nullable(Int32), +dns_rcode Nullable(Int32), +dns_qdcount Nullable(Int32), +dns_ancount Nullable(Int32), +dns_nscount Nullable(Int32), +dns_arcount Nullable(Int32), +dns_qname String, +dns_qtype Nullable(Int32), +dns_qclass Nullable(Int32), +dns_cname String, +dns_sub Nullable(Int32), +dns_rr String, +dns_response_latency_ms Nullable(Int32), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +mail_protocol_type String, +mail_account String, +mail_from_cmd String, +mail_to_cmd String, +mail_from String, +mail_password String, +mail_to String, +mail_cc String, +mail_bcc String, +mail_subject String, +mail_subject_charset String, +mail_attachment_name String, +mail_attachment_name_charset String, +mail_starttls_flag Nullable(Int32), +mail_eml_file String, +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +sip_bye_reason String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand()); + + + +alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; +alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +direction String, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +ip_protocol LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +sip_bye_reason String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,decoded_as,data_center, device_group,recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +direction String, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +client_ip String, +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +server_ip String, +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +ip_protocol LowCardinality(String), +sip_call_id String, +sip_originator_description String, +sip_responder_description String, +sip_user_agent String, +sip_server String, +sip_originator_sdp_connect_ip String, +sip_originator_sdp_media_port Nullable(Int32), +sip_originator_sdp_media_type String, +sip_originator_sdp_content String, +sip_responder_sdp_connect_ip String, +sip_responder_sdp_media_port Nullable(Int32), +sip_responder_sdp_media_type String, +sip_responder_sdp_content String, +sip_duration_s Nullable(Int32), +sip_bye String, +sip_bye_reason String, +rtp_payload_type_c2s Nullable(Int32), +rtp_payload_type_s2c Nullable(Int32), +rtp_pcap_path String, +rtp_originator_dir Nullable(Int32), +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64 MATERIALIZED toUnixTimestamp(now()), +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +direction String, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_ip_tags Array(String), +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_ip_tags Array(String), +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_fqdn_tags Array(String), +server_domain String, +app_transition String,  +app LowCardinality(String), +app_category String, +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,proxy_action,decoded_as,data_center, device_group,recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_cluster ( +recv_time Int64, +log_id UInt64, +decoded_as String, +session_id UInt64, +start_timestamp_ms DateTime64(3), +end_timestamp_ms DateTime64(3), +duration_ms Int32, +tcp_handshake_latency_ms Nullable(Int32), +ingestion_time Int64, +processing_time Int64, +insert_time Int64, +device_id String, +out_link_id Nullable(Int32), +in_link_id Nullable(Int32), +device_tag String, +data_center String, +device_group String, +sled_ip String, +address_type Int32, +direction String, +vsys_id Int32, +t_vsys_id Int32, +flags Int64, +flags_identify_info String, +c2s_ttl Nullable(Int32), +s2c_ttl Nullable(Int32), +security_rule_list Array(Int64), +security_action String, +monitor_rule_list Array(Int64), +shaping_rule_list Array(Int64), +proxy_rule_list Array(Int64), +statistics_rule_list Array(Int64), +sc_rule_list Array(Int64), +sc_rsp_raw Array(Int64), +sc_rsp_decrypted Array(Int64), +proxy_action String, +proxy_pinning_status Nullable(Int32), +proxy_intercept_status Nullable(Int32), +proxy_passthrough_reason String, +proxy_client_side_latency_ms Nullable(Int32), +proxy_server_side_latency_ms Nullable(Int32), +proxy_client_side_version String, +proxy_server_side_version String, +proxy_cert_verify Nullable(Int32), +proxy_intercept_error String, +monitor_mirrored_pkts Nullable(Int32), +monitor_mirrored_bytes Nullable(Int32), +client_ip String, +client_ip_tags Array(String), +client_port Int32, +client_os_desc String, +client_geolocation LowCardinality(String), +client_country String, +client_super_administrative_area String, +client_administrative_area String, +client_sub_administrative_area String, +client_asn Nullable(Int64), +subscriber_id String, +imei String, +imsi String, +phone_number String, +apn String, +server_ip String, +server_ip_tags Array(String), +server_port Int32, +server_os_desc String, +server_geolocation LowCardinality(String), +server_country String, +server_super_administrative_area String, +server_administrative_area String, +server_sub_administrative_area String, +server_asn Nullable(Int64), +server_fqdn String, +server_fqdn_tags Array(String), +server_domain String, +app_transition String,  +app LowCardinality(String), +app_category String, +app_debug_info String, +app_content String, +app_extra_info String, +fqdn_category_list Array(Int64), +ip_protocol LowCardinality(String), +decoded_path LowCardinality(String), +http_url String, +http_host String, +http_request_line String, +http_response_line String, +http_request_body String, +http_response_body String, +http_proxy_flag Nullable(Int32), +http_sequence Nullable(Int32), +http_cookie String, +http_referer String, +http_user_agent String, +http_request_content_length Nullable(Int64), +http_request_content_type String, +http_response_content_length Nullable(Int64), +http_response_content_type String, +http_set_cookie String, +http_version String, +http_status_code Nullable(Int32), +http_response_latency_ms Nullable(Int32), +http_session_duration_ms Nullable(Int32), +http_action_file_size Nullable(Int64), +doh_url String, +doh_host String, +doh_request_line String, +doh_response_line String, +doh_cookie String, +doh_referer String, +doh_user_agent String, +doh_content_length String, +doh_content_type String, +doh_set_cookie String, +doh_version String, +doh_message_id Int64, +doh_qr Nullable(Int64), +doh_opcode Nullable(Int64), +doh_aa Nullable(Int64), +doh_tc Nullable(Int64), +doh_rd Nullable(Int64), +doh_ra Nullable(Int64), +doh_rcode Nullable(Int64), +doh_qdcount Nullable(Int64), +doh_ancount Nullable(Int64), +doh_nscount Nullable(Int64), +doh_arcount Nullable(Int64), +doh_qname String, +doh_qtype Nullable(Int64), +doh_qclass Nullable(Int64), +doh_cname String, +doh_sub Nullable(Int64), +doh_rr String, +sent_pkts Int64, +received_pkts Int64, +sent_bytes Int64, +received_bytes Int64, +tcp_c2s_ip_fragments Nullable(Int64), +tcp_s2c_ip_fragments Nullable(Int64), +tcp_c2s_lost_bytes Nullable(Int64), +tcp_s2c_lost_bytes Nullable(Int64), +tcp_c2s_o3_pkts Nullable(Int64), +tcp_s2c_o3_pkts Nullable(Int64), +tcp_c2s_rtx_pkts Nullable(Int64), +tcp_s2c_rtx_pkts Nullable(Int64), +tcp_c2s_rtx_bytes Nullable(Int64), +tcp_s2c_rtx_bytes Nullable(Int64), +tcp_rtt_ms Nullable(Int32), +tcp_client_isn Nullable(Int64), +tcp_server_isn Nullable(Int64), +packet_capture_file String, +in_src_mac String, +out_src_mac String, +in_dest_mac String, +out_dest_mac String, +encapsulation String, +dup_traffic_flag Nullable(Int32), +tunnel_id_list Array(Int64), +tunnel_endpoint_a_desc String, +tunnel_endpoint_b_desc String +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand()); + + +-- tsg_galaxy_v3.security_event_materialized_view +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster +TO tsg_galaxy_v3.security_event_local +( + recv_time Int64, + log_id UInt64, + decoded_as String, + session_id UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int32, + tcp_handshake_latency_ms Nullable(Int32), + ingestion_time Int64, + processing_time Int64, + -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), + device_id String, + out_link_id Nullable(Int32), + in_link_id Nullable(Int32), + device_tag String, + data_center String, + device_group String, + sled_ip String, + address_type Int32, + direction String, + vsys_id Int32, + t_vsys_id Int32, + flags Int64, + flags_identify_info String, + c2s_ttl Nullable(Int32), + s2c_ttl Nullable(Int32), + security_rule_list Array(Int64), + security_action String, + monitor_rule_list Array(Int64), + shaping_rule_list Array(Int64), + proxy_rule_list Array(Int64), + statistics_rule_list Array(Int64), + sc_rule_list Array(Int64), + sc_rsp_raw Array(Int64), + sc_rsp_decrypted Array(Int64), + proxy_action String, + proxy_pinning_status Nullable(Int32), + proxy_intercept_status Nullable(Int32), + proxy_passthrough_reason String, + proxy_client_side_latency_ms Nullable(Int32), + proxy_server_side_latency_ms Nullable(Int32), + proxy_client_side_version String, + proxy_server_side_version String, + proxy_cert_verify Nullable(Int32), + proxy_intercept_error String, + monitor_mirrored_pkts Nullable(Int32), + monitor_mirrored_bytes Nullable(Int32), + client_ip String, + client_ip_tags Array(String), + client_port Int32, + client_os_desc String, + client_geolocation LowCardinality(String), + client_country String, + client_super_administrative_area String, + client_administrative_area String, + client_sub_administrative_area String, + client_asn Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + server_ip String, + server_ip_tags Array(String), + server_port Int32, + server_os_desc String, + server_geolocation LowCardinality(String), + server_country String, + server_super_administrative_area String, + server_administrative_area String, + server_sub_administrative_area String, + server_asn Nullable(Int64), + server_fqdn String, + server_fqdn_tags Array(String), + server_domain String, + app_transition String, + app LowCardinality(String), + app_category String, + app_debug_info String, + app_content String, + app_extra_info String, + fqdn_category_list Array(Int64), + ip_protocol LowCardinality(String), + decoded_path LowCardinality(String), + dns_message_id Nullable(Int32), + dns_qr Nullable(Int32), + dns_opcode Nullable(Int32), + dns_aa Nullable(Int32), + dns_tc Nullable(Int32), + dns_rd Nullable(Int32), + dns_ra Nullable(Int32), + dns_rcode Nullable(Int32), + dns_qdcount Nullable(Int32), + dns_ancount Nullable(Int32), + dns_nscount Nullable(Int32), + dns_arcount Nullable(Int32), + dns_qname String, + dns_qtype Nullable(Int32), + dns_qclass Nullable(Int32), + dns_cname String, + dns_sub Nullable(Int32), + dns_rr String, + dns_response_latency_ms Nullable(Int32), + http_url String, + http_host String, + http_request_line String, + http_response_line String, + http_request_body String, + http_response_body String, + http_proxy_flag Nullable(Int32), + http_sequence Nullable(Int32), + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length Nullable(Int64), + http_request_content_type String, + http_response_content_length Nullable(Int64), + http_response_content_type String, + http_set_cookie String, + http_version String, + http_status_code Nullable(Int32), + http_response_latency_ms Nullable(Int32), + http_session_duration_ms Nullable(Int32), + http_action_file_size Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_handshake_latency_ms Nullable(Int32), + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + ssl_esni_flag Nullable(Int32), + ssl_ech_flag Nullable(Int32), + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_handshake_latency_ms Nullable(Int32), + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + mail_protocol_type String, + mail_account String, + mail_from_cmd String, + mail_to_cmd String, + mail_from String, + mail_password String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_starttls_flag Nullable(Int32), + mail_eml_file String, + ftp_account String, + ftp_url String, + ftp_link_type String, + quic_version String, + quic_sni String, + quic_user_agent String, + rdp_cookie String, + rdp_security_protocol String, + rdp_client_channels String, + rdp_keyboard_layout String, + rdp_client_version String, + rdp_client_name String, + rdp_client_product_id String, + rdp_desktop_width String, + rdp_desktop_height String, + rdp_requested_color_depth String, + rdp_certificate_type String, + rdp_certificate_count Nullable(Int32), + rdp_certificate_permanent Nullable(Int32), + rdp_encryption_level String, + rdp_encryption_method String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Nullable(Int32), + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Nullable(Int32), + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Nullable(Int32), + sip_bye String, + sip_bye_reason String, + rtp_payload_type_c2s Nullable(Int32), + rtp_payload_type_s2c Nullable(Int32), + rtp_pcap_path String, + rtp_originator_dir Nullable(Int32), + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + sent_pkts Int64, + received_pkts Int64, + sent_bytes Int64, + received_bytes Int64, + tcp_c2s_ip_fragments Nullable(Int64), + tcp_s2c_ip_fragments Nullable(Int64), + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_rtt_ms Nullable(Int32), + tcp_client_isn Nullable(Int64), + tcp_server_isn Nullable(Int64), + packet_capture_file String, + in_src_mac String, + out_src_mac String, + in_dest_mac String, + out_dest_mac String, + encapsulation String, + dup_traffic_flag Nullable(Int32), + tunnel_id_list Array(Int64), + tunnel_endpoint_a_desc String, + tunnel_endpoint_b_desc String +) +AS +SELECT + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + direction, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + c2s_ttl, + s2c_ttl, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + proxy_rule_list, + statistics_rule_list, + sc_rule_list, + sc_rsp_raw, + sc_rsp_decrypted, + proxy_action, + proxy_pinning_status, + proxy_intercept_status, + proxy_passthrough_reason, + proxy_client_side_latency_ms, + proxy_server_side_latency_ms, + proxy_client_side_version, + proxy_server_side_version, + proxy_cert_verify, + proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_ip_tags, + client_port, + client_os_desc, + client_geolocation, + client_country, + client_super_administrative_area, + client_administrative_area, + client_sub_administrative_area, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + apn, + server_ip, + server_ip_tags, + server_port, + server_os_desc, + server_geolocation, + server_country, + server_super_administrative_area, + server_administrative_area, + server_sub_administrative_area, + server_asn, + server_fqdn, + server_fqdn_tags, + server_domain, + app_transition, + app, + app_category, + app_debug_info, + app_content, + app_extra_info, + fqdn_category_list, + ip_protocol, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + ssl_esni_flag, + ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_starttls_flag, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + sip_bye_reason, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + encapsulation, + dup_traffic_flag, + tunnel_id_list, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +FROM tsg_galaxy_v3.session_record_local +WHERE empty(security_rule_list) = 0 +; + +-- tsg_galaxy_v3.monitor_event_materialized_view +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster +TO tsg_galaxy_v3.monitor_event_local +( + recv_time Int64, + log_id UInt64, + decoded_as String, + session_id UInt64, + start_timestamp_ms DateTime64(3), + end_timestamp_ms DateTime64(3), + duration_ms Int32, + tcp_handshake_latency_ms Nullable(Int32), + ingestion_time Int64, + processing_time Int64, + -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()), + device_id String, + out_link_id Nullable(Int32), + in_link_id Nullable(Int32), + device_tag String, + data_center String, + device_group String, + sled_ip String, + address_type Int32, + direction String, + vsys_id Int32, + t_vsys_id Int32, + flags Int64, + flags_identify_info String, + c2s_ttl Nullable(Int32), + s2c_ttl Nullable(Int32), + security_rule_list Array(Int64), + security_action String, + monitor_rule_list Array(Int64), + shaping_rule_list Array(Int64), + proxy_rule_list Array(Int64), + statistics_rule_list Array(Int64), + sc_rule_list Array(Int64), + sc_rsp_raw Array(Int64), + sc_rsp_decrypted Array(Int64), + proxy_action String, + proxy_pinning_status Nullable(Int32), + proxy_intercept_status Nullable(Int32), + proxy_passthrough_reason String, + proxy_client_side_latency_ms Nullable(Int32), + proxy_server_side_latency_ms Nullable(Int32), + proxy_client_side_version String, + proxy_server_side_version String, + proxy_cert_verify Nullable(Int32), + proxy_intercept_error String, + monitor_mirrored_pkts Nullable(Int32), + monitor_mirrored_bytes Nullable(Int32), + client_ip String, + client_ip_tags Array(String), + client_port Int32, + client_os_desc String, + client_geolocation LowCardinality(String), + client_country String, + client_super_administrative_area String, + client_administrative_area String, + client_sub_administrative_area String, + client_asn Nullable(Int64), + subscriber_id String, + imei String, + imsi String, + phone_number String, + apn String, + server_ip String, + server_ip_tags Array(String), + server_port Int32, + server_os_desc String, + server_geolocation LowCardinality(String), + server_country String, + server_super_administrative_area String, + server_administrative_area String, + server_sub_administrative_area String, + server_asn Nullable(Int64), + server_fqdn String, + server_fqdn_tags Array(String), + server_domain String, + app_transition String, + app LowCardinality(String), + app_category String, + app_debug_info String, + app_content String, + app_extra_info String, + fqdn_category_list Array(Int64), + ip_protocol LowCardinality(String), + decoded_path LowCardinality(String), + dns_message_id Nullable(Int32), + dns_qr Nullable(Int32), + dns_opcode Nullable(Int32), + dns_aa Nullable(Int32), + dns_tc Nullable(Int32), + dns_rd Nullable(Int32), + dns_ra Nullable(Int32), + dns_rcode Nullable(Int32), + dns_qdcount Nullable(Int32), + dns_ancount Nullable(Int32), + dns_nscount Nullable(Int32), + dns_arcount Nullable(Int32), + dns_qname String, + dns_qtype Nullable(Int32), + dns_qclass Nullable(Int32), + dns_cname String, + dns_sub Nullable(Int32), + dns_rr String, + dns_response_latency_ms Nullable(Int32), + http_url String, + http_host String, + http_request_line String, + http_response_line String, + http_request_body String, + http_response_body String, + http_proxy_flag Nullable(Int32), + http_sequence Nullable(Int32), + http_cookie String, + http_referer String, + http_user_agent String, + http_request_content_length Nullable(Int64), + http_request_content_type String, + http_response_content_length Nullable(Int64), + http_response_content_type String, + http_set_cookie String, + http_version String, + http_status_code Nullable(Int32), + http_response_latency_ms Nullable(Int32), + http_session_duration_ms Nullable(Int32), + http_action_file_size Nullable(Int64), + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_handshake_latency_ms Nullable(Int32), + ssl_ja3_hash String, + ssl_ja3s_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + ssl_esni_flag Nullable(Int32), + ssl_ech_flag Nullable(Int32), + dtls_cookie String, + dtls_version String, + dtls_sni String, + dtls_san String, + dtls_cn String, + dtls_handshake_latency_ms Nullable(Int32), + dtls_ja3_fingerprint String, + dtls_ja3_hash String, + dtls_cert_issuer String, + dtls_cert_subject String, + mail_protocol_type String, + mail_account String, + mail_from_cmd String, + mail_to_cmd String, + mail_from String, + mail_password String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_starttls_flag Nullable(Int32), + mail_eml_file String, + ftp_account String, + ftp_url String, + ftp_link_type String, + quic_version String, + quic_sni String, + quic_user_agent String, + rdp_cookie String, + rdp_security_protocol String, + rdp_client_channels String, + rdp_keyboard_layout String, + rdp_client_version String, + rdp_client_name String, + rdp_client_product_id String, + rdp_desktop_width String, + rdp_desktop_height String, + rdp_requested_color_depth String, + rdp_certificate_type String, + rdp_certificate_count Nullable(Int32), + rdp_certificate_permanent Nullable(Int32), + rdp_encryption_level String, + rdp_encryption_method String, + ssh_version String, + ssh_auth_success String, + ssh_client_version String, + ssh_server_version String, + ssh_cipher_alg String, + ssh_mac_alg String, + ssh_compression_alg String, + ssh_kex_alg String, + ssh_host_key_alg String, + ssh_host_key String, + ssh_hassh String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Nullable(Int32), + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Nullable(Int32), + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration_s Nullable(Int32), + sip_bye String, + sip_bye_reason String, + rtp_payload_type_c2s Nullable(Int32), + rtp_payload_type_s2c Nullable(Int32), + rtp_pcap_path String, + rtp_originator_dir Nullable(Int32), + stratum_cryptocurrency String, + stratum_mining_pools String, + stratum_mining_program String, + stratum_mining_subscribe String, + sent_pkts Int64, + received_pkts Int64, + sent_bytes Int64, + received_bytes Int64, + tcp_c2s_ip_fragments Nullable(Int64), + tcp_s2c_ip_fragments Nullable(Int64), + tcp_c2s_lost_bytes Nullable(Int64), + tcp_s2c_lost_bytes Nullable(Int64), + tcp_c2s_o3_pkts Nullable(Int64), + tcp_s2c_o3_pkts Nullable(Int64), + tcp_c2s_rtx_pkts Nullable(Int64), + tcp_s2c_rtx_pkts Nullable(Int64), + tcp_c2s_rtx_bytes Nullable(Int64), + tcp_s2c_rtx_bytes Nullable(Int64), + tcp_rtt_ms Nullable(Int32), + tcp_client_isn Nullable(Int64), + tcp_server_isn Nullable(Int64), + packet_capture_file String, + in_src_mac String, + out_src_mac String, + in_dest_mac String, + out_dest_mac String, + encapsulation String, + dup_traffic_flag Nullable(Int32), + tunnel_id_list Array(Int64), + tunnel_endpoint_a_desc String, + tunnel_endpoint_b_desc String +) +AS +SELECT + recv_time, + log_id, + decoded_as, + session_id, + start_timestamp_ms, + end_timestamp_ms, + duration_ms, + tcp_handshake_latency_ms, + ingestion_time, + processing_time, + -- insert_time, + device_id, + out_link_id, + in_link_id, + device_tag, + data_center, + device_group, + sled_ip, + address_type, + direction, + vsys_id, + t_vsys_id, + flags, + flags_identify_info, + c2s_ttl, + s2c_ttl, + security_rule_list, + security_action, + monitor_rule_list, + shaping_rule_list, + proxy_rule_list, + statistics_rule_list, + sc_rule_list, + sc_rsp_raw, + sc_rsp_decrypted, + proxy_action, + proxy_pinning_status, + proxy_intercept_status, + proxy_passthrough_reason, + proxy_client_side_latency_ms, + proxy_server_side_latency_ms, + proxy_client_side_version, + proxy_server_side_version, + proxy_cert_verify, + proxy_intercept_error, + monitor_mirrored_pkts, + monitor_mirrored_bytes, + client_ip, + client_ip_tags, + client_port, + client_os_desc, + client_geolocation, + client_country, + client_super_administrative_area, + client_administrative_area, + client_sub_administrative_area, + client_asn, + subscriber_id, + imei, + imsi, + phone_number, + apn, + server_ip, + server_ip_tags, + server_port, + server_os_desc, + server_geolocation, + server_country, + server_super_administrative_area, + server_administrative_area, + server_sub_administrative_area, + server_asn, + server_fqdn, + server_fqdn_tags, + server_domain, + app_transition, + app, + app_category, + app_debug_info, + app_content, + app_extra_info, + fqdn_category_list, + ip_protocol, + decoded_path, + dns_message_id, + dns_qr, + dns_opcode, + dns_aa, + dns_tc, + dns_rd, + dns_ra, + dns_rcode, + dns_qdcount, + dns_ancount, + dns_nscount, + dns_arcount, + dns_qname, + dns_qtype, + dns_qclass, + dns_cname, + dns_sub, + dns_rr, + dns_response_latency_ms, + http_url, + http_host, + http_request_line, + http_response_line, + http_request_body, + http_response_body, + http_proxy_flag, + http_sequence, + http_cookie, + http_referer, + http_user_agent, + http_request_content_length, + http_request_content_type, + http_response_content_length, + http_response_content_type, + http_set_cookie, + http_version, + http_status_code, + http_response_latency_ms, + http_session_duration_ms, + http_action_file_size, + ssl_version, + ssl_sni, + ssl_san, + ssl_cn, + ssl_handshake_latency_ms, + ssl_ja3_hash, + ssl_ja3s_hash, + ssl_cert_issuer, + ssl_cert_subject, + ssl_esni_flag, + ssl_ech_flag, + dtls_cookie, + dtls_version, + dtls_sni, + dtls_san, + dtls_cn, + dtls_handshake_latency_ms, + dtls_ja3_fingerprint, + dtls_ja3_hash, + dtls_cert_issuer, + dtls_cert_subject, + mail_protocol_type, + mail_account, + mail_from_cmd, + mail_to_cmd, + mail_from, + mail_password, + mail_to, + mail_cc, + mail_bcc, + mail_subject, + mail_subject_charset, + mail_attachment_name, + mail_attachment_name_charset, + mail_starttls_flag, + mail_eml_file, + ftp_account, + ftp_url, + ftp_link_type, + quic_version, + quic_sni, + quic_user_agent, + rdp_cookie, + rdp_security_protocol, + rdp_client_channels, + rdp_keyboard_layout, + rdp_client_version, + rdp_client_name, + rdp_client_product_id, + rdp_desktop_width, + rdp_desktop_height, + rdp_requested_color_depth, + rdp_certificate_type, + rdp_certificate_count, + rdp_certificate_permanent, + rdp_encryption_level, + rdp_encryption_method, + ssh_version, + ssh_auth_success, + ssh_client_version, + ssh_server_version, + ssh_cipher_alg, + ssh_mac_alg, + ssh_compression_alg, + ssh_kex_alg, + ssh_host_key_alg, + ssh_host_key, + ssh_hassh, + sip_call_id, + sip_originator_description, + sip_responder_description, + sip_user_agent, + sip_server, + sip_originator_sdp_connect_ip, + sip_originator_sdp_media_port, + sip_originator_sdp_media_type, + sip_originator_sdp_content, + sip_responder_sdp_connect_ip, + sip_responder_sdp_media_port, + sip_responder_sdp_media_type, + sip_responder_sdp_content, + sip_duration_s, + sip_bye, + sip_bye_reason, + rtp_payload_type_c2s, + rtp_payload_type_s2c, + rtp_pcap_path, + rtp_originator_dir, + stratum_cryptocurrency, + stratum_mining_pools, + stratum_mining_program, + stratum_mining_subscribe, + sent_pkts, + received_pkts, + sent_bytes, + received_bytes, + tcp_c2s_ip_fragments, + tcp_s2c_ip_fragments, + tcp_c2s_lost_bytes, + tcp_s2c_lost_bytes, + tcp_c2s_o3_pkts, + tcp_s2c_o3_pkts, + tcp_c2s_rtx_pkts, + tcp_s2c_rtx_pkts, + tcp_c2s_rtx_bytes, + tcp_s2c_rtx_bytes, + tcp_rtt_ms, + tcp_client_isn, + tcp_server_isn, + packet_capture_file, + in_src_mac, + out_src_mac, + in_dest_mac, + out_dest_mac, + encapsulation, + dup_traffic_flag, + tunnel_id_list, + tunnel_endpoint_a_desc, + tunnel_endpoint_b_desc +FROM tsg_galaxy_v3.session_record_local +WHERE empty(monitor_rule_list) = 0 +; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record_local on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int32, + timestamp_us UInt64, + egress_action Int32, + job_id String, + sled_ip String, + device_group String, + traffic_link_id Int32, + source_ip String, + source_port Nullable(Int32), + destination_ip String, + destination_port Nullable(Int32), + packet String, + packet_length Int32, + measurements String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id,job_id,recv_time,timestamp_us); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.datapath_telemetry_record on cluster ck_cluster ( + log_id UInt64, + recv_time Int64, + vsys_id Int32, + timestamp_us UInt64, + egress_action Int32, + job_id String, + sled_ip String, + device_group String, + traffic_link_id Int32, + source_ip String, + source_port Nullable(Int32), + destination_ip String, + destination_port Nullable(Int32), + packet String, + packet_length Int32, + measurements String +) +ENGINE = Distributed('ck_cluster', + 'tsg_galaxy_v3', + 'datapath_telemetry_record_local', + rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.traffic_sketch_metric_local on cluster ck_cluster +( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + device_id String, + device_group String, + data_center String, + direction String, + ip_protocol String, + client_ip String, + server_ip String, + internal_ip String, + external_ip String, + client_country String, + server_country String, + client_asn Nullable(Int64), + server_asn Nullable(Int64), + server_fqdn String, + server_domain String, + app String, + app_category String, + c2s_ttl Nullable(Int32), + s2c_ttl Nullable(Int32), + c2s_link_id Nullable(Int32), + s2c_link_id Nullable(Int32), + sessions Int64, + bytes Int64, + sent_bytes Int64, + received_bytes Int64, + pkts Int64, + sent_pkts Int64, + received_pkts Int64, + asymmetric_c2s_flows Int64, + asymmetric_s2c_flows Int64, + c2s_fragments Int64, + s2c_fragments Int64, + c2s_tcp_lost_bytes Int64, + s2c_tcp_lost_bytes Int64, + c2s_tcp_retransmitted_pkts Int64, + s2c_tcp_retransmitted_pkts Int64 +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(toDate(recv_time)) +ORDER BY (vsys_id, + direction, + ip_protocol, + app, + client_ip, + recv_time); + + CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.traffic_sketch_metric on cluster ck_cluster +( + log_id UInt64, + recv_time Int64, + vsys_id Int64, + device_id String, + device_group String, + data_center String, + direction String, + ip_protocol String, + client_ip String, + server_ip String, + internal_ip String, + external_ip String, + client_country String, + server_country String, + client_asn Nullable(Int64), + server_asn Nullable(Int64), + server_fqdn String, + server_domain String, + app String, + app_category String, + c2s_ttl Nullable(Int32), + s2c_ttl Nullable(Int32), + c2s_link_id Nullable(Int32), + s2c_link_id Nullable(Int32), + sessions Int64, + bytes Int64, + sent_bytes Int64, + received_bytes Int64, + pkts Int64, + sent_pkts Int64, + received_pkts Int64, + asymmetric_c2s_flows Int64, + asymmetric_s2c_flows Int64, + c2s_fragments Int64, + s2c_fragments Int64, + c2s_tcp_lost_bytes Int64, + s2c_tcp_lost_bytes Int64, + c2s_tcp_retransmitted_pkts Int64, + s2c_tcp_retransmitted_pkts Int64 +) +ENGINE = Distributed('ck_cluster', + 'tsg_galaxy_v3', + 'traffic_sketch_metric_local', + rand()); \ No newline at end of file diff --git a/clickhouse/tsg_olap_clickhouse_ddl_check.sql b/clickhouse/tsg_olap_clickhouse_ddl_check.sql new file mode 100644 index 0000000..2bf242c --- /dev/null +++ b/clickhouse/tsg_olap_clickhouse_ddl_check.sql @@ -0,0 +1,22 @@ +SELECT log_id, recv_time, vsys_id, assessment_date, lot_number, file_name, assessment_file, assessment_type, features, `size`, file_checksum_sha +FROM tsg_galaxy_v3.assessment_event where recv_time >= toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time = toUnixTimestamp('2030-01-01 00:00:00') AND recv_time 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_time_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_filename_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_partfile_troubleshooting_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_time_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_filename_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} +alter 'index_partfile_assessment_file_bucket',{NAME => 'data',METADATA => {'DFS_REPLICATION' => '1'}},{NAME => 'meta', METADATA => {'DFS_REPLICATION' => '1'}} + +EOF diff --git a/hos/bucket_upgrade.sh b/hos/bucket_upgrade.sh new file mode 100644 index 0000000..740b4da --- /dev/null +++ b/hos/bucket_upgrade.sh @@ -0,0 +1,6 @@ +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_policy_capture_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_rtp_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_http_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_eml_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' + +curl -X DELETE http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_file_bucket -H 'token:{{ hos_token }}' \ No newline at end of file diff --git a/hos/create_bucket.sh b/hos/create_bucket.sh new file mode 100644 index 0000000..0473632 --- /dev/null +++ b/hos/create_bucket.sh @@ -0,0 +1,8 @@ +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_policy_capture_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_rtp_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_http_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:64*hbase服务器数' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/traffic_eml_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/troubleshooting_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/assessment_file_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-ttl:30' -H 'x-hos-replication:1' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/knowledge_base_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:单机为1集群为2' +curl -X PUT http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/report_snapshot_bucket -H 'token:{{ hos_token }}' -H 'x-hos-region-count:16' -H 'x-hos-wal:open' -H 'x-hos-replication:单机为1集群为2' \ No newline at end of file diff --git a/hos/galaxy-hos-service-24.09.yml b/hos/galaxy-hos-service-24.09.yml new file mode 100644 index 0000000..b4ce78f --- /dev/null +++ b/hos/galaxy-hos-service-24.09.yml @@ -0,0 +1,97 @@ +#服务端口 +server: + port: 8186 + max-http-header-size: 20MB + tomcat: + max-threads: 400 +#tomcat缓存大小，单位KB系统默认10M，配置10g +tomcat: + cacheMaxSize: 1000000 +#hbase参数 +hbase: + zookeeperQuorum: 192.168.44.11:2181,192.168.44.14:2181,192.168.44.15:2181 + zookeeperPort: 2181 + zookeeperNodeParent: /hbase + clientRetriesNumber: 9 + rpcTimeout: 100000 + connectPool: 10 + clientWriteBuffer: 10485760 + clientKeyValueMaxsize: 1073741824 + mobThreshold: 10485760 + #part的最大数量 + maxParts: 100000 + #每次获取的part数 + getPartBatch: 10 + #hbase索引表前缀，前缀为以下的都为索引表 + timeIndexTablePrefix: index_time_ + filenameIndexTablePrefix: index_filename_ + partFileIndexTablePrefix: index_partfile_ + systemBucketMeta: system:bucket_meta + #创建表的分区数 + regionCount: 16 + filenameHead: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + partHead: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + #获取文件大小的目录 + dataPath: /hbase + #hadoop集群namenode节点，单机为单个ip，集群为ip1,ip2 + hadoopNameNodes: 192.168.44.10,192.168.44.11 + #副本数，单机为1，集群为2 + hadoopReplication: 2 + #hadoop端口 + hadoopPort: 9000 + hadoopUser: root + hadoopNameServices: ns1 + hadoopNameNodesNs1: nn1,nn2 + asyncPut: 0 +#是否打开验证，0打开，打开需要使用S3身份验证或者token访问服务 +auth: + open: 0 + #http访问使用的token + token: ENC(vknRT6U4I739rLIha9CvojM+4uFyXZLEYpO2HZayLnRak1HPW0K2yZ3vnQBA2foo) + #s3验证 + s3: + accesskey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) + secretkey: ENC(FUQDvVP+zqCiwHQhXcRvbw==) +hos: + #文件大小阈值 + maxFileSize: 5073741800 + #大文件阈值 + uploadThreshold: 104857600 + #长连接超时时间 + keepAliveTimeout: 60000 + #批量删除对象的最大数量 + deleteMultipleNumber: 1000 + #获取对象列表等操作的最大值 + maxResultLimit: 100000 + #分块上传的最大分块数 + maxPartNumber: 10000 + #追加上传的最大次数 + maxAppendNumber: 100000 + #是否快速上传 + isQuickUpload: 0 + #是否快速下载文件，1打开，hbase内存小于20G的集群设为0 + isQuickDownloadFile: 0 + #用户白名单（hbase的namespace），获取存储配额 + users: default + #是否打开限流,0:关闭，1:打开 + openRateLimiter: 0 + #限流每秒请求数 + rateLimiterQps: 20000 +#设置上传文件大小的最大值 +spring: + servlet: + multipart: + max-file-size: 5GB + max-request-size: 5GB +#Prometheus参数 + application: + name: HosServiceApplication +#Prometheus参数 +management: + endpoints: + web: + exposure: + include: '*' + metrics: + tags: + application: ${spring.application.name} \ No newline at end of file diff --git a/hos/hosutil/config.properties b/hos/hosutil/config.properties new file mode 100644 index 0000000..fc486bf --- /dev/null +++ b/hos/hosutil/config.properties @@ -0,0 +1,21 @@ +qgw.serverAddr=http://{{ vrrp_instance.default.virtual_ipaddress }}:9999 +hos.serverAddr=http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098 +hos.token={{ hos_token }} +kafka.server={{ groups.kafka[0] }}:9092 +#延迟时间，校验多少秒之前的文件，单位秒 +check.time.delay=180 +hos.traffic.buckets=traffic_policy_capture_file_bucket,traffic_rtp_file_bucket,traffic_http_file_bucket,traffic_eml_file_bucket +kafka.traffic.topics=TRAFFIC-POLICY-CAPTURE-FILE-STREAM-RECORD,TRAFFIC-RTP-FILE-STREAM-RECORD,TRAFFIC-HTTP-FILE-STREAM-RECORD,TRAFFIC-EML-FILE-STREAM-RECORD +kafka.troubleshooting.topic=TROUBLESHOOTING-FILE-STREAM-RECORD +file.chunk.combiner.window.time=15000 +traffic.file.count=10 +threads=1 +max.threads=10 +print.out.interval=1000 +http.max.total=100 +http.default.max.per.route=100 +http.connect.timeout=5000 +http.connection.request.timeout=10000 +http.socket.timeout=-1 +hos.log.types=security_event,monitor_event,proxy_event,session_record,voip_record,assessment_event,transaction_record,troubleshooting +hos.log.types.file.types.url.fields=security_event:http-http_response_body&http_request_body,pcap-packet_capture_file&rtp_pcap_path,eml-mail_eml_file;proxy_event:http-http_response_body&http_request_body;session_record:http-http_response_body&http_request_body,pcap-packet_capture_file&rtp_pcap_path,eml-mail_eml_file;voip_record:pcap-rtp_pcap_path;assessment_event:other-assessment_file;transaction_record:http-http_response_body&http_request_body,eml-mail_eml_file;monitor_event:http-http_response_body&http_request_body,pcap-packet_capture_file&rtp_pcap_path,eml-mail_eml_file \ No newline at end of file diff --git a/hos/hosutil/galaxy-hos-util-1.4.jar b/hos/hosutil/galaxy-hos-util-1.4.jar new file mode 100644 index 0000000..9b05a71 Binary files /dev/null and b/hos/hosutil/galaxy-hos-util-1.4.jar differ diff --git a/hos/hosutil/hosutil.sh b/hos/hosutil/hosutil.sh new file mode 100644 index 0000000..e74c7ff --- /dev/null +++ b/hos/hosutil/hosutil.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +version="1.4" +jar="galaxy-hos-util-$version.jar" + +usage() { + cat <&2; usage; exit 1 ;; + :) echo "Option -$OPTARG requires an argument" >&2; usage; exit 1 ;; + esac +done + +case "$operation" in + download) download ;; + upload) upload ;; + check) check ;; + combiner) combiner ;; + version) echo $version ;; + *) usage; exit 1 ;; +esac + diff --git a/kafka/kafka-operation.sh b/kafka/kafka-operation.sh new file mode 100644 index 0000000..fbcbf11 --- /dev/null +++ b/kafka/kafka-operation.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +LOCAL_IP=127.0.0.1:9094 + +ZK_SERVER=127.0.0.1:2181/kafka +KAFKA_SERVER=127.0.0.1:9092 +PARTITIONS=1 + +usage() { + echo "------------------------------------------------------------------------" + echo -e "生产数据\n" + echo "交互式：kafka-operation.sh producer MOCK-DATA-SESSION-RECORD" | column -t + echo "读取文件：kafka-operation.sh producer MOCK-DATA-SESSION-RECORD < mock_data.txt" | column -t + echo "------------------------------------------------------------------------" + echo -e "消费数据\n" + echo "从当前消费：kafka-operation.sh consumer MOCK-DATA-SESSION-RECORD" | column -t + echo "从头消费：kafka-operation.sh consumer-begin MOCK-DATA-SESSION-RECORD" | column -t + echo "------------------------------------------------------------------------" + echo -e "创建Topic\n" + echo "kafka-operation.sh create-topic [topic name] [partition num] [replication num]" | column -t + echo "样例：kafka-operation.sh create-topic MOCK-DATA-SESSION-RECORD 3 1" | column -t + echo "------------------------------------------------------------------------" + echo -e "删除Topic\n" + echo "kafka-operation.sh delete-topic [topic name]" | column -t + echo "样例：kafka-operation.sh delete-topic MOCK-DATA-SESSION-RECORD" | column -t + echo "------------------------------------------------------------------------" + echo -e "查看Topic列表\n" + echo "kafka-operation.sh list" | column -t + echo "------------------------------------------------------------------------" + echo -e "查看Topic详情\n" + echo "kafka-operation.sh desc-topic [topic name]" | column -t + echo "样例：kafka-operation.sh desc-topic MOCK-DATA-SESSION-RECORD" | column -t + echo "------------------------------------------------------------------------" + echo -e "查看消费组列表\n" + echo "kafka-operation.sh groups" | column -t + echo "------------------------------------------------------------------------" + echo -e "查看消费组详情\n" + echo "kafka-operation.sh desc-group [consumer group name]" | column -t + echo "样例：kafka-operation.sh desc-group etl-session-record-kafka-to-kafka" | column -t + echo "------------------------------------------------------------------------" + echo -e "增加Topic分区\n" + echo "kafka-operation.sh add-partition [topic name] [new partition num]" | column -t + echo "样例：kafka-operation.sh add-partition MOCK-DATA-SESSION-RECORD 5" | column -t + echo "------------------------------------------------------------------------" + echo -e "Topic Leader平衡\n" + echo "kafka-operation.sh election-leader" | column -t + echo "------------------------------------------------------------------------" + echo -e "查看Quotas\n" + echo "kafka-operation.sh desc-quota" | column -t + echo "------------------------------------------------------------------------" + echo -e "增加Quotas\n" + echo "kafka-operation.sh add-quota [quatos rule] [user] [client.id]" | column -t + echo "kafka-operation.sh add-quota 'producer_byte_rate=10485760' tsg_olap SESSION-RECORD" | column -t + echo "------------------------------------------------------------------------" + echo -e "删除Quotas\n" + echo "kafka-operation.sh delete-quota [quatos rule] [user] [client.id]" | column -t + echo "kafka-operation.sh delete-quota 'producer_byte_rate' tsg_olap SESSION-RECORD" | column -t + echo "------------------------------------------------------------------------" +} + +if [ $# -eq 0 ]; then + usage + exit 0 +fi + +case $1 in + producer) + kafka-console-producer.sh --producer.config $KAFKA_HOME/config/producer.properties --broker-list $LOCAL_IP --topic $2 + ;; + consumer) + kafka-console-consumer.sh --consumer.config $KAFKA_HOME/config/consumer.properties --bootstrap-server $LOCAL_IP --topic $2 + ;; + consumer-begin) + kafka-console-consumer.sh --consumer.config $KAFKA_HOME/config/consumer.properties --from-beginning --bootstrap-server $LOCAL_IP --topic $2 + ;; + create-topic) + kafka-topics.sh --if-not-exists --create --bootstrap-server $KAFKA_SERVER --topic $2 --partitions $3 --replication-factor $4 + ;; + delete-topic) + kafka-topics.sh --if-exists --delete --bootstrap-server $KAFKA_SERVER --topic $2 + ;; + list) + kafka-topics.sh --list --bootstrap-server $KAFKA_SERVER + ;; + desc-topic) + kafka-topics.sh --if-exists --bootstrap-server $KAFKA_SERVER --describe --topic $2 + ;; + groups) + kafka-consumer-groups.sh --all-groups --all-topics --list --bootstrap-server $KAFKA_SERVER + ;; + desc-group) + kafka-consumer-groups.sh --bootstrap-server $KAFKA_SERVER --describe --group $2 + ;; + add-partition) + kafka-topics.sh --if-exists --bootstrap-server $KAFKA_SERVER --alter --topic $2 --partitions $3 + ;; + election-leader) + kafka-leader-election.sh --bootstrap-server $KAFKA_SERVER --all-topic-partitions --election-type PREFERRED + ;; + desc-quota) + kafka-configs.sh --bootstrap-server $KAFKA_SERVER --describe --entity-type users --entity-type clients + ;; + add-quota) + kafka-configs.sh --bootstrap-server $KAFKA_SERVER --alter --add-config $2 --entity-type users --entity-name $3 --entity-type clients --entity-name $4 + ;; + delete-quota) + kafka-configs.sh --bootstrap-server $KAFKA_SERVER --alter --delete-config $2 --entity-type users --entity-name $3 --entity-type clients --entity-name $4 + ;; + *) + usage +esac diff --git a/kafka/kafka_server_jaas.conf b/kafka/kafka_server_jaas.conf new file mode 100644 index 0000000..d331d00 --- /dev/null +++ b/kafka/kafka_server_jaas.conf @@ -0,0 +1,8 @@ +KafkaServer { + org.apache.kafka.common.security.plain.PlainLoginModule required + username="admin" + password="galaxy2019" + user_admin="galaxy2019" + user_firewall="galaxy2019" + user_olap="galaxy2019"; +}; diff --git a/monitoring-templates/TSG-OLAP-AlertRule.xlsx b/monitoring-templates/TSG-OLAP-AlertRule.xlsx new file mode 100644 index 0000000..4f5ca47 Binary files /dev/null and b/monitoring-templates/TSG-OLAP-AlertRule.xlsx differ diff --git a/monitoring-templates/TSG-OLAP-Dashboard.json b/monitoring-templates/TSG-OLAP-Dashboard.json new file mode 100644 index 0000000..d1113d1 --- /dev/null +++ b/monitoring-templates/TSG-OLAP-Dashboard.json @@ -0,0 +1,20299 @@ +[ + { + "name":"[24.09] OLAP Component Status Template", + "type":"template", + "varType":0, + "param":{ + "chartShare":"none", + "variables":[ + { + "regex":"", + "expression":"", + "customOptions":"Zookeeper,Kafka,HDFS,Yarn,HBase,Druid,Clickhouse,Arangodb,Nacos,Mariadb", + "show":true, + "name":"Component", + "allOption":true, + "type":"custom", + "multi":true + }, + { + "regex":"", + "expression":"", + "customOptions":"TSG-OLAP,CN-OLAP,TSG-DT", + "show":true, + "name":"Project", + "allOption":false, + "type":"custom", + "multi":false + } + ], + "report":{ + "schedule":{ + "repeat":1, + "etime":"", + "stime":"", + "type":2, + "nums":[] + }, + "receivers":[], + "enable":false, + "range":{ + "unit":"day", + "interval":1, + "type":"previous" + } + }, + "refresh":0, + "defaultTimeRange":4 + }, + "remark":"The template for bigdata component status.", + "charts":[ + { + "name":"Zookeeper Status", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":0, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Component", + "varValue":"Zookeeper", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"1.0", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Disk IO Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0-100)", + "weight":1, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "val":"80", + "color":"#d64f40", + "id":"a85bb03", + "value":80 + } + ], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The maximum percentage of DISK IO in each asset.", + "x":"8.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(asset)(rate(node_disk_io_time_seconds_total{olap_node_exporter=~\".*zookeeper.*\",project=\"$Project\"}[5m]))*100", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107313, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Outstanding Requests", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":2, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "min", + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "val":"100", + "color":"#d64f40", + "id":"62fbf61", + "value":100 + } + ], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of queued requests in each Data Center. This goes up when the server receives more requests than it can process. When count > 100.", + "x":"8.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (zookeeper_outstanding_requests{module=~\".*Zookeeper\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107314, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Zookeeper Connections/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":3, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "min", + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The number of connections per second in each Data Center", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (rate(zookeeper_connections{module=~\".*Zookeeper\",project=\"$Project\"}[5m]))", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107316, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Avg Latency", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"milliseconds", + "weight":4, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "val":"100000", + "color":"#d64f40", + "id":"58f8e1ad", + "value":100000 + } + ], + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Amount of time it takes for the server to respond to a client request (since the server was started) in each data center. When latency > 10 (Ticks).", + "x":"4.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (rate(zookeeper_latency_avg_ms{module=~\".*Zookeeper\",project=\"$Project\"}[5m]))", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107319, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Sent Packets/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"packets/sec(IEC)", + "weight":5, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of client packets sent (responses and notifications) in each data center.", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(asset) (rate(zookeeper_packets_sent{module=~\".*Zookeeper\",project=\"$Project\"}[5m]))\n", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107321, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Received Packets/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"packets/sec(IEC)", + "weight":6, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "total", + "min", + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of client requests (typically operations) received in each data center.", + "x":"4.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(asset) (rate(zookeeper_packets_received{module=~\".*Zookeeper\",project=\"$Project\"}[5m]))\n", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107323, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Flink on Yarn Status", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":8, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Component", + "varValue":"Yarn", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"9.38", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Resourcemanager Memory Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":9, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#C4162AFF", + "id":"884bf8a", + "value":0.8 + } + ], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The Hadoop Yarn ResourceManager nodes memory Utilization.", + "x":"0.0", + "y":"3.0", + "elements":[ + { + "expression":"sum by(asset) (Hadoop_ResourceManager_MemHeapUsedM{module=~\".*Yarn-ResourceManager\",project=\"$Project\"}/Hadoop_ResourceManager_MemHeapMaxM{module=~\".*Yarn-ResourceManager\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107344, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"ResourceManager CPU Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":10, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "val":"0.7", + "color":"#d64f40", + "id":"68f1c6c2", + "value":0.8 + } + ], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The Hadoop Yarn ResourceManager nodes CPU Utilization.", + "x":"4.0", + "y":"3.0", + "elements":[ + { + "expression":"sum by (asset)(java_lang_OperatingSystem_ProcessCpuLoad{module=~\".*Yarn-ResourceManager\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107345, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"NodeManager CPU Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":11, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "val":"0.7", + "color":"#d64f40", + "id":"a927eeb", + "value":0.8 + } + ], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The Hadoop Yarn NodeManager nodes CPU Utilization.", + "x":"4.0", + "y":"5.0", + "elements":[ + { + "expression":"sum by(asset) (java_lang_OperatingSystem_ProcessCpuLoad{module=~\".*Yarn-NodeManager\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107348, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"NodeManager Memory Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":12, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "val":"0.7", + "color":"#d64f40", + "id":"2022c819", + "value":0.8 + } + ], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The Hadoop Yarn NodeManager nodes memory Utilization.", + "x":"0.0", + "y":"5.0", + "elements":[ + { + "expression":"sum by (asset)(Hadoop_NodeManager_MemHeapUsedM{module=~\".*Yarn-NodeManager\",project=\"$Project\"}/Hadoop_NodeManager_MemHeapMaxM{module=~\".*Yarn-NodeManager\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107349, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"NodeManager RpcProcessing AvgTime", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"milliseconds", + "weight":13, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#7f4ee8", + "id":"b6ef207" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The average RPC call time of the NodeManager.", + "x":"0.0", + "y":"7.0", + "elements":[ + { + "expression":"sum by(asset)(Hadoop_NodeManager_RpcProcessingTimeAvgTime{module=~\".*Yarn-NodeManager\",project=\"$Project\",name=\"RpcActivityForPort9923\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107350, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"ResourceManager GC Count", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"none", + "weight":14, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#bd00f2", + "id":"da71837" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of GC times of the ResourceManager node", + "x":"8.0", + "y":"3.0", + "elements":[ + { + "expression":"sum by(asset,name) (irate(java_lang_GarbageCollector_CollectionCount{module=~\".*Yarn-ResourceManager\",project=\"$Project\",name=~\"PS MarkSweep|PS Scavenge\"}[5m]))", + "legend":"{{asset}}-{{name}}", + "name":"A", + "orderNum":0, + "id":107351, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"NodeManager GC Count", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"none", + "weight":15, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#bd00f2", + "id":"2781ca5" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of GC times of the NodeManager node", + "x":"8.0", + "y":"5.0", + "elements":[ + { + "expression":"sum by(asset,name) (irate(java_lang_GarbageCollector_CollectionCount{module=~\".*Yarn-NodeManager\",project=\"$Project\",name=~\"PS MarkSweep|PS Scavenge\"}[5m]))", + "legend":"{{asset}}-{{name}}", + "name":"A", + "orderNum":0, + "id":107352, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"ResourceManager GC Duration", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"milliseconds", + "weight":16, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#bd00f2", + "id":"40f9b83" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"ResourceManager GC duration", + "x":"4.0", + "y":"7.0", + "elements":[ + { + "expression":"sum by(asset,name) (irate(java_lang_GarbageCollector_CollectionTime{module=~\".*Yarn-ResourceManager\",project=\"$Project\",name=~\"PS MarkSweep|PS Scavenge\"}[5m]))", + "legend":"{{asset}}-{{name}}", + "name":"A", + "orderNum":0, + "id":107353, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"NodeManager GC Duration", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"milliseconds", + "weight":17, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#bd00f2", + "id":"393c4a57" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"ResourceManager GC duration", + "x":"8.0", + "y":"7.0", + "elements":[ + { + "expression":"sum by(asset,name) (irate(java_lang_GarbageCollector_CollectionTime{module=~\".*Yarn-NodeManager\",project=\"$Project\",name=~\"PS MarkSweep|PS Scavenge\"}[5m]))", + "legend":"{{asset}}-{{name}}", + "name":"A", + "orderNum":0, + "id":107354, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Flink Job Restarts", + "span":"12.0", + "height":"3.0", + "type":"Line chart", + "unit":"short", + "weight":18, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "link":"", + "thresholds":[ + { + "color":"#036eb5", + "id":"7eca5e9d" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"max by(datacenter, job_name) ((flink_jobmanager_job_numRestarts{project=\"$Project\"}) - (flink_jobmanager_job_numRestarts{project=\"$Project\"} offset 30s))", + "legend":"{{datacenter}}:{{job_name}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Mariadb Status", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":19, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Component", + "varValue":"Mariadb", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"5.19", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Number Of Connections", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":20, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#7447b7", + "id":"625b5574" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"mysql_global_status_threads_connected{project=~\"$Project\"}", + "legend":"{{asset}} threads_connected", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"mysql_global_status_threads_cached{project=~\"$Project\"}", + "legend":"{{asset}} threads_cached", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"mysql_global_status_threads_running{project=~\"$Project\"}", + "legend":"{{asset}} threads_running", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Mariadb Slow Queries", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":21, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#7447b7", + "id":"6c3025e" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"increase(mysql_global_status_slow_queries{project=\"$Project\"}[2m])", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Mariadb Throughput", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":22, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#7447b7", + "id":"fb2dad1" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"rate(mysql_global_status_questions{project=\"$Project\"}[2m])", + "legend":"{{asset}} Read rate", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum without (command) (\nrate(mysql_global_status_commands_total{command=~\"insert|update|delete\",project=\"$Project\"}[2m])\n)", + "legend":"{{asset}} Write rate", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Mariadb Connection Refused", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":23, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#7447b7", + "id":"90a5db3" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"mysql_global_status_aborted_connects{project=\"$Project\"}", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Nacos Status", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":24, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Component", + "varValue":"Nacos", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"22.77", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"QPS", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":25, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Nacos  queries per second", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(method,uri) (rate(http_server_requests_seconds_count{uri=~'/v1/cs/configs|/nacos/v1/ns/instance|/nacos/v1/ns/health', module=~\".*Nacos\",project=\"$Project\"}[5m]))", + "legend":"{{method}}", + "name":"A", + "orderNum":0, + "id":107325, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Request Errors", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":26, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The requests failed counter per second of each http method.", + "x":"4.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(method,uri) (irate(http_server_requests_seconds_count{exception!='None',module=~\".*Nacos\",project=\"$Project\"}[5m]))", + "legend":"{{method}}-{{uri}}", + "name":"A", + "orderNum":0, + "id":107326, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Memory Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":27, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The memory usage in each asset", + "x":"8.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (jvm_memory_used_bytes{module=~\".*Nacos\",project=\"$Project\"}) / sum by(asset) (jvm_memory_max_bytes{module=~\".*Nacos\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107327, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"CPU Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":28, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The CPU usage in each asset", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(asset) (system_cpu_usage{module=~\".*Nacos\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107328, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Kafka Status", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":29, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Component", + "varValue":"Kafka", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"18.58", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Failed Fetch Requests/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":30, + "param":{ + "rightYAxis":{ + "elementNames":[ + "A" + ], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "rightYAxis":true, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of (producer|consumer|follower) requests per second in each Data Center.", + "x":"4.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(asset) (kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"FailedFetchRequestsPerSec\",topic=\"\",module=~\".*Kafka\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"In Bytes/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes(IEC)", + "weight":31, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "total", + "min", + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Collecting and processing Traffic Records with incoming byte rate in each Data Center.", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesInPerSec\",topic=\"\",module=~\".*Kafka\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107330, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Out Bytes/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes(IEC)", + "weight":32, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "total", + "min", + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Collecting and processing Traffic Records with outgoing byte rate in each Data Center.", + "x":"4.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesOutPerSec\",topic=\"\",module=~\".*Kafka\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107332, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Failed Produce Request/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":33, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of data produce requests from producers that brokers failed to process for this topic in each Data Center.", + "x":"8.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(datacenter) (kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"FailedProduceRequestsPerSec\",topic=\"\"})", + "legend":"{{datacenter}}", + "name":"A", + "orderNum":0, + "id":107334, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Asset Disk IO Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0-100)", + "weight":34, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "thresholds":[ + { + "val":"80", + "color":"#d64f40", + "id":"329f354", + "value":80 + } + ], + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The maxium percentage of DISK IO in each data center and asset.", + "x":"8.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset)(rate(node_disk_io_time_seconds_total{olap_node_exporter=~\".*kafka.*\",project=\"$Project\"}[5m]))*100", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107336, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Avg Zookeeper Request Latency", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"milliseconds", + "weight":35, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "min", + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The average zookeeper request latency(ms) in each data center and asset.", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"avg by (asset)(kafka_server_ZooKeeperClientMetrics_Mean{module=~\".*Kafka\",project=\"$Project\"}) ", + "legend":"{{datacenter}} - {{asset}}", + "name":"A", + "orderNum":0, + "id":107337, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Server Socket Requests/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":82, + "param":{ + "rightYAxis":{ + "elementNames":[ + "A" + ], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "rightYAxis":true, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of (producer|consumer|follower) requests per second in each Data Center.", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (kafka_server_socket_server_metrics_request_rate{module=~\".*Kafka\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Druid Status", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":36, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Component", + "varValue":"Druid", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"26.96", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Index Status", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":37, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"3.0", + "elements":[ + { + "expression":"avg(druid_index_running_task_num{project=\"$Project\"})", + "legend":"Running", + "name":"A", + "orderNum":0, + "id":107356, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"avg(druid_index_waiting_task_num{project=\"$Project\"})", + "legend":"Waiting", + "name":"B", + "orderNum":1, + "id":107357, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"avg(druid_index_pending_task_num{project=\"$Project\"})", + "legend":"Pending", + "name":"C", + "orderNum":2, + "id":107358, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Compact Status", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":38, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"4.0", + "y":"3.0", + "elements":[ + { + "expression":"avg(druid_compact_waiting_task_num{project=\"$Project\"})", + "legend":"Waiting", + "name":"A", + "orderNum":0, + "id":107359, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"avg(druid_compact_pending_task_num{project=\"$Project\"})", + "legend":"Pending", + "name":"B", + "orderNum":1, + "id":107360, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"DataSource Segment Size", + "span":"4.0", + "height":"3.0", + "type":"Table", + "unit":"bytes(IEC)", + "weight":39, + "param":{ + "thresholds":[], + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "columns":[ + { + "unit":2, + "display":"{{A.$legend}}", + "show":true, + "title":"DataSource", + "error":false + }, + { + "unit":7, + "display":"{{A.$value}}", + "show":true, + "title":"Segment Size", + "error":false + } + ], + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "indexs":"", + "tableOptions":{ + "showTableHeader":"enabled", + "pagination":"enabled" + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null", + "statistics":"last" + }, + "remark":"The total segment size in each Datasource.", + "x":"4.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(dataSource)(coordinator_segment_size{service=\"coordinator\",module=~\".*Druid\",project=\"$Project\"})", + "legend":"{{dataSource}}", + "name":"A", + "orderNum":0, + "id":107362, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Query CPU Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0-100)", + "weight":40, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "thresholds":[ + { + "val":"0.8", + "color":"#d64f40", + "id":"24a1af1", + "value":80 + } + ], + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "min", + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The CPU usage in each asset.", + "x":"0.0", + "y":"5.0", + "elements":[ + { + "expression":"(1-avg(irate(node_cpu_seconds_total{mode=\"idle\",olap_node_exporter=~\".*druid_query.*\",project=\"$Project\"}[5m])) by(asset)) * 100", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107363, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Query Memory Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":41, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "thresholds":[ + { + "val":"0.8", + "color":"#d64f40", + "id":"3f12ca98", + "value":80 + } + ], + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The memory usage in each asset. Runs Coordinator and Overlord processes, which manages data availability and ingestion.", + "x":"8.0", + "y":"3.0", + "elements":[ + { + "expression":"1 - (node_memory_MemAvailable_bytes{olap_node_exporter=~\".*druid_query.*\",project=\"$Project\"} / node_memory_MemTotal_bytes{olap_node_exporter=~\".*druid_query.*\",project=\"$Project\"}) ", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107364, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Data Memory Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":42, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "thresholds":[ + { + "val":"80", + "color":"#d64f40", + "id":"f8b7db1", + "value":80 + } + ], + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The memory usage in each asset.  Runs Historical and MiddleManager processes, which executes ingestion workloads and stores all queryable data.", + "x":"4.0", + "y":"5.0", + "elements":[ + { + "expression":"1 - (node_memory_MemAvailable_bytes{olap_node_exporter=~\".*druid_data.*\",project=\"$Project\"} / node_memory_MemTotal_bytes{olap_node_exporter=~\".*druid_data.*\",project=\"$Project\"}) ", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107365, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Data CPU Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0-100)", + "weight":43, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "thresholds":[ + { + "val":"90", + "color":"#d64f40", + "id":"1ae5f1c2", + "value":80 + } + ], + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"connected" + }, + "remark":"The CPU usage in each asset.", + "x":"8.0", + "y":"5.0", + "elements":[ + { + "expression":"(1-avg(irate(node_cpu_seconds_total{mode=\"idle\",olap_node_exporter=~\".*druid_data.*\",project=\"$Project\"}[5m])) by(asset)) * 100", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107366, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"DataSource Segment Count", + "span":"4.0", + "height":"3.0", + "type":"Table", + "unit":"bytes(IEC)", + "weight":44, + "param":{ + "thresholds":[], + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "columns":[ + { + "unit":2, + "display":"{{A.$legend}}", + "show":true, + "title":"DataSource", + "error":false + }, + { + "unit":2, + "display":"{{A.$value}}", + "show":true, + "title":"Segment Count", + "error":false + } + ], + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "indexs":"", + "tableOptions":{ + "showTableHeader":"enabled", + "pagination":"enabled" + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null", + "statistics":"last" + }, + "remark":"The segment number in each Datasource.", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by (dataSource) (coordinator_segment_count{service=\"coordinator\",module=~\".*Druid\",project=\"$Project\"})", + "legend":"{{dataSource}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Historical Node Segement Used", + "span":"4.0", + "height":"3.0", + "type":"Table", + "unit":"bytes(IEC)", + "weight":45, + "param":{ + "thresholds":[], + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "columns":[ + { + "unit":2, + "display":"{{A.$legend}}", + "show":true, + "title":"Asset", + "error":false + }, + { + "unit":7, + "display":"{{A.$value}}", + "show":true, + "title":"Used", + "error":false + } + ], + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "indexs":"", + "tableOptions":{ + "showTableHeader":"enabled", + "pagination":"enabled" + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null", + "statistics":"last" + }, + "remark":"The total segment size in each Datasource.", + "x":"8.0", + "y":"0.0", + "elements":[ + { + "expression":" sum by (asset) (historical_segment_used{service=\"historical\",project=\"$Project\"}) ", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Historical Segment Used percent", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":46, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#a032ef", + "id":"b84ca4e" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The percent of the process's temporary cache data on disk", + "x":"0.0", + "y":"7.0", + "elements":[ + { + "expression":"sum by (asset) (historical_segment_used_percent{service=\"historical\"}) ", + "legend":"{{asset}}-{{server}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Clickhouse Status", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":47, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Component", + "varValue":"Clickhouse", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"43.35", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Insert Rows/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":48, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "total", + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of rows INSERTed to all tables", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (irate(clickhouse_inserted_rows_total{module=~\".*Clickhouse\",project=\"$Project\"}[5m]))", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107369, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Insert Bytes/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":49, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "total", + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of bytes (uncompressed) INSERTed to all tables", + "x":"4.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (irate(clickhouse_inserted_bytes_total{module=~\".*Clickhouse\",project=\"$Project\"}[5m]))\n", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107370, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Merged Rate/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":50, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "min", + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The merging data parts per second in each Data center.", + "x":"0.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by(asset) (irate(clickhouse_merge{module=~\".*Clickhouse\",project=\"$Project\"}[5m]))", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107372, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Bad Requests/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":51, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The bad requests per second in each Data center.", + "x":"4.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(asset) (irate(bad_requests_total{module=~\".*Clickhouse\",project=\"$Project\"}[5m]))", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107373, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Memory Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":52, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The memory usage in each asset.", + "x":"4.0", + "y":"6.0", + "elements":[ + { + "expression":"1 - (node_memory_MemAvailable_bytes{olap_node_exporter=~\".*clickhouse.*\",project=\"$Project\"} / node_memory_MemTotal_bytes{olap_node_exporter=~\".*clickhouse.*\",project=\"$Project\"}) ", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107374, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"CPU Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":53, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The CPU usage in each asset.", + "x":"0.0", + "y":"6.0", + "elements":[ + { + "expression":"(1-avg(irate(node_cpu_seconds_total{mode=\"idle\",olap_node_exporter=~\".*clickhouse.*\",project=\"$Project\"}[5m])) by(asset))", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107375, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"TCP Connection Time", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"seconds(s)", + "weight":54, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "max", + "avg" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The current Tcp Connection time in each asset.", + "x":"8.0", + "y":"2.0", + "elements":[ + { + "expression":"max by(asset) (ck_connect_time{olap_node_exporter=~\".*clickhouse.*\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107376, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Active Query Count", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":55, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "max", + "avg" + ], + "show":true, + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#7dd1ed", + "id":"2ab3239d" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The current number of queries in each asset.", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"max by(asset) (ck_processes_count{olap_node_exporter=~\".*clickhouse.*\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107377, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Mutiation part Count", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":56, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The current number of mutiation part in each asset.", + "x":"4.0", + "y":"4.0", + "elements":[ + { + "expression":"max by(asset)(clickhouse_part_mutation{module=~\".*Clickhouse\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107378, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Asset Disk IO Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0-100)", + "weight":57, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The maxium percentage of DISK IO in each data center and asset.", + "x":"8.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by(asset)(rate(node_disk_io_time_seconds_total{olap_node_exporter=~\".*clickhouse.*\",project=\"$Project\"}[5m]))*100", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107379, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Expired Parts Count", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":58, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "max", + "avg" + ], + "show":true, + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#7dd1ed", + "id":"567833a8" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The current number of queries in each asset.", + "x":"8.0", + "y":"0.0", + "elements":[ + { + "expression":"ck_expired_parts_count{olap_node_exporter=~\".*clickhouse.*\",project=\"$Project\"}", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"HBase Status", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":59, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Component", + "varValue":"HBase", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"42.35", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Request/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":60, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"3524cce" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of requests per second of each type,Read/Write/Total", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by (datacenter) (irate(Hadoop_HBase_totalRequestCount{project=\"$Project\"}[5m]))", + "legend":"{{datacenter}} - Total", + "name":"A", + "orderNum":0, + "id":107380, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (datacenter)(irate(Hadoop_HBase_readRequestCount{project=\"$Project\"}[5m]))", + "legend":"{{datacenter}} - Read", + "name":"B", + "orderNum":1, + "id":107381, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (datacenter)(irate(Hadoop_HBase_writeRequestCount{project=\"$Project\"}[5m]))", + "legend":"{{datacenter}} - Write", + "name":"C", + "orderNum":2, + "id":107382, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Region Count", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":61, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"1978b85f" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of HBase cluster regions", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by (datacenter) (Hadoop_HBase_regionCount{module=~\".*HBase.*\",project=\"$Project\"})", + "legend":"{{datacenter}}", + "name":"A", + "orderNum":0, + "id":107383, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"RitRegion(Unhealthy) Count", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":62, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"266c1168" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of HBase cluster rit(Region in transition) regions.", + "x":"4.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by (datacenter) (Hadoop_HBase_ritCount{module=~\".*HBase.*\",project=\"$Project\"})", + "legend":"{{datacenter}}", + "name":"A", + "orderNum":0, + "id":107384, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Memory Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":63, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#C4162AFF", + "id":"45b41f1", + "value":80 + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The Memory usage in each asset.", + "x":"8.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (Hadoop_HBase_MemHeapUsedM{module=~\".*HBase-HregionServer\",project=\"$Project\"}) / sum by(asset) (Hadoop_HBase_MemHeapMaxM{module=~\".*HBase-HregionServer\",project=\"$Project\"})\n", + "legend":"{{asset}}-HregionServer", + "name":"A", + "orderNum":0, + "id":107385, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(asset) (Hadoop_HBase_MemHeapUsedM{module=~\".*HBase-Hmaster\",project=\"$Project\"}) / sum by(asset) (Hadoop_HBase_MemHeapMaxM{module=~\".*HBase-Hmaster\",project=\"$Project\"})\n", + "legend":"{{asset}}-Hmaster", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"CPU Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":64, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#C4162AFF", + "id":"8ff1639", + "value":80 + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The CPU usage in each asset.", + "x":"4.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (java_lang_OperatingSystem_ProcessCpuLoad{module=~\".*HBase-HregionServer\",project=\"$Project\"})", + "legend":"{{asset}}-HregionServer", + "name":"A", + "orderNum":0, + "id":107386, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(asset) (java_lang_OperatingSystem_ProcessCpuLoad{module=~\".*HBase-Hmaster\",project=\"$Project\"})", + "legend":"{{asset}}-Hmaster", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Request Queue", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":65, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"3c37e86" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of requests in the HBase request queue.", + "x":"8.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(datacenter)(Hadoop_HBase_numCallsInGeneralQueue{name=\"RegionServer\",module=~\".*HBase.*\",project=\"$Project\"})", + "legend":"{{datacenter}}", + "name":"A", + "orderNum":0, + "id":107387, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Slow Request", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":66, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"1cc27af" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of slow requests (put,append,get) in the HBase cluster.", + "x":"0.0", + "y":"4.0", + "elements":[ + { + "expression":"sum(irate(Hadoop_HBase_slowPutCount{name=\"RegionServer\",sub=\"Server\",module=~\".*HBase.*\",project=\"$Project\"}[5m])) by (datacenter)", + "legend":"put-{{datacenter}}", + "name":"A", + "orderNum":0, + "id":107388, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(irate(Hadoop_HBase_slowAppendCount{name=\"RegionServer\",sub=\"Server\",module=~\".*HBase.*\",project=\"$Project\"}[5m])) by (datacenter)", + "legend":"append-{{datacenter}}", + "name":"B", + "orderNum":1, + "id":107389, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(irate(Hadoop_HBase_slowGetCount{name=\"RegionServer\",sub=\"Server\",module=~\".*HBase.*\",project=\"$Project\"}[5m])) by (datacenter)", + "legend":"get-{{datacenter}}", + "name":"C", + "orderNum":2, + "id":107390, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Memstore Size", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes(IEC)", + "weight":67, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"6802e74" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Data size in the HBase node memory", + "x":"4.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by (asset) (Hadoop_HBase_metric_memStoreSize{module=~\".*HBase.*\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107391, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"GC Count/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"none", + "weight":68, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"160af06" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"GC times per second of the HBase process.", + "x":"8.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by (datacenter)(irate(Hadoop_HBase_GcCount{module=~\".*HBase.*\",project=\"$Project\"}[5m]))", + "legend":"GCCount-{{datacenter}}", + "name":"A", + "orderNum":0, + "id":107392, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (datacenter)(irate(Hadoop_HBase_GcCountParNew{module=~\".*HBase.*\",project=\"$Project\"}[5m]))", + "legend":"ParNewGcCount-{{datacenter}}", + "name":"B", + "orderNum":1, + "id":107393, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (datacenter)(irate(Hadoop_HBase_GcCountConcurrentMarkSweep{module=~\".*HBase.*\",project=\"$Project\"}[5m]))", + "legend":"CMSGcCount-{{datacenter}}", + "name":"C", + "orderNum":2, + "id":107394, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"GC Time", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"milliseconds", + "weight":69, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"34dde05e" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The GC duration of the HBase process.", + "x":"0.0", + "y":"6.0", + "elements":[ + { + "expression":"sum by (datacenter)(irate(Hadoop_HBase_GcCount{module=~\".*HBase.*\",project=\"$Project\"}[5m]))", + "legend":"GC-{{datacenter}}", + "name":"A", + "orderNum":0, + "id":107395, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (datacenter)(irate(Hadoop_HBase_GcCountParNew{module=~\".*HBase.*\",project=\"$Project\"}[5m]))", + "legend":"ParNewGc-{{datacenter}}", + "name":"B", + "orderNum":1, + "id":107396, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (datacenter)(irate(Hadoop_HBase_GcCountConcurrentMarkSweep{module=~\".*HBase.*\",project=\"$Project\"}[5m]))", + "legend":"CMSGc-{{datacenter}}", + "name":"C", + "orderNum":2, + "id":107397, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Put Request By HOS Table/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":70, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"7b439968" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of put requests for hos table per second.", + "x":"4.0", + "y":"6.0", + "elements":[ + { + "expression":"sum by (table) (irate(Hadoop_HBase_metric_putCount{namespace=\"default\",type=\"data\",module=~\".*HBase.*\",project=\"$Project\"}[5m]))", + "legend":"{{table}}", + "name":"A", + "orderNum":0, + "id":107398, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Get Request By HOS Table/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":71, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"5b700717" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of get requests for hos table per second.", + "x":"8.0", + "y":"6.0", + "elements":[ + { + "expression":"sum by (table) (irate(Hadoop_HBase_metric_getCount{namespace=\"default\",type=\"data\",module=~\".*HBase.*\",project=\"$Project\"}[5m]))", + "legend":"{{table}}", + "name":"A", + "orderNum":0, + "id":107399, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"HOS TTL Scan Count By Table /sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":72, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#0c4dff", + "id":"2e8ec6" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of scan times in ttl for hos table per second.", + "x":"0.0", + "y":"8.0", + "elements":[ + { + "expression":"sum by (table) (irate(Hadoop_HBase_metric_scanCount{namespace=\"default\",type=\"data\",module=~\".*HBase.*\",project=\"$Project\"}[5m]))", + "legend":"{{table}}", + "name":"A", + "orderNum":0, + "id":107401, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Hadoop HDFS Status", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":73, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Component", + "varValue":"HDFS", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "showHeader":true, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"36.15", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Storage Capacity Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0-100)", + "weight":74, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#C4162AFF", + "id":"a0c3ac9", + "value":80 + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Hadoop cluster storage capacity Usage.", + "x":"4.0", + "y":"4.0", + "elements":[ + { + "expression":"(sum by(datacenter) (Hadoop_NameNode_PercentUsed{module=~\".*HDFS-NameNode\",project=\"$Project\"}))/2\n", + "legend":"{{datacenter}}", + "name":"A", + "orderNum":0, + "id":107403, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Namenode Memory Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":75, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#1c1984", + "id":"6107b19" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The Memory usage in each Namenode.", + "x":"4.0", + "y":"0.0", + "elements":[ + { + "expression":"Hadoop_NameNode_MemHeapUsedM{module=~\".*HDFS-NameNode\",project=\"$Project\"}/Hadoop_NameNode_MemHeapMaxM{module=~\".*HDFS-NameNode\",project=\"$Project\"}", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107404, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Datanode Memory Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":76, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#1c1984", + "id":"f850c09" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The Memory usage in each Datanode.", + "x":"4.0", + "y":"2.0", + "elements":[ + { + "expression":"Hadoop_DataNode_MemHeapUsedM{module=~\".*HDFS-DataNode\",project=\"$Project\"}/Hadoop_DataNode_MemHeapMaxM{module=~\".*HDFS-DataNode\",project=\"$Project\"}\n", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107405, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Missing Blocks", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":77, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#1c1984", + "id":"7a84367" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of blocks with missing in the Hadoop cluster.", + "x":"8.0", + "y":"2.0", + "elements":[ + { + "expression":"(sum by(datacenter) (Hadoop_NameNode_NumberOfMissingBlocks{module=~\".*HDFS-NameNode\",project=\"$Project\"}))/2", + "legend":"{{datacenter}}", + "name":"A", + "orderNum":0, + "id":107406, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Namenode CPU Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":78, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#1c1984", + "id":"d62e843" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The CPU usage in each Namenode.", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"java_lang_OperatingSystem_ProcessCpuLoad{module=~\".*HDFS-NameNode\",project=\"$Project\"}", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107407, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Corrupt Blocks", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":79, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#1c1984", + "id":"41dda95c" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of blocks with corrupt replicas in the Hadoop cluster.", + "x":"8.0", + "y":"0.0", + "elements":[ + { + "expression":"(sum by(datacenter) (Hadoop_NameNode_CorruptBlocks{module=~\".*HDFS-NameNode\",project=\"$Project\"}))/2", + "legend":"{{datacenter}}", + "name":"B", + "orderNum":0, + "id":107408, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Under Replicated Blocks", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":80, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[ + { + "color":"#1c1984", + "id":"b43381d" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of blocks with lose replicated in the Hadoop cluster.", + "x":"0.0", + "y":"4.0", + "elements":[ + { + "expression":"(sum by(datacenter) (Hadoop_NameNode_UnderReplicatedBlocks{module=~\".*HDFS-NameNode\",project=\"$Project\"}))/2\n", + "legend":"{{datacenter}}", + "name":"C", + "orderNum":0, + "id":107409, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Datanode CPU Usage", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":81, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":true, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#1c1984", + "id":"5742c3da" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The CPU usage in each DataNode.", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"java_lang_OperatingSystem_ProcessCpuLoad{module=~\".*HDFS-DataNode\",project=\"$Project\"}", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":107410, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + } + ], + "children":[] + }, + { + "name":"[24.09] OLAP Service Status Template", + "type":"template", + "varType":0, + "param":{ + "chartShare":"none", + "variables":[ + { + "regex":"", + "expression":"", + "customOptions":"TSG-OLAP,CN-OLAP,TSG-DT", + "show":true, + "name":"Project", + "allOption":false, + "type":"custom", + "multi":false + } + ], + "report":{ + "schedule":{ + "repeat":1, + "etime":"", + "stime":"", + "type":2, + "nums":[] + }, + "receivers":[], + "enable":false, + "range":{ + "unit":"day", + "interval":1, + "type":"previous" + } + }, + "refresh":0, + "defaultTimeRange":4 + }, + "remark":"", + "charts":[ + { + "name":"Summary", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":0, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Project", + "varValue":"TSG-DT", + "operator":"notEqual" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"1.0", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"QGW Response Latency", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"seconds(s)", + "weight":1, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "val":"60", + "color":"#d64f40", + "id":"2d432055", + "value":90 + } + ], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"QGW service response latency. Used to evaluate whether there is a slow query currently.", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(rate(http_server_requests_seconds_sum{module=~\".*Galaxy-qgw-service\",status!~\"5..\",project=\"$Project\"}[5m]))/sum(rate(http_server_requests_seconds_count{module=~\".*Galaxy-qgw-service\", status!~\"5..\",project=\"$Project\"}[5m]))", + "legend":"AVG", + "name":"A", + "orderNum":0, + "id":108725, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"max(http_server_requests_seconds_max{module=\"Galaxy-qgw-service\", status!~\"5..\",project=\"$Project\"})", + "legend":" MAX", + "name":"B", + "orderNum":1, + "id":108726, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"HOS Put Object Requests/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":2, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The number of put objects per second. Include total, success, and failure.", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(irate(http_server_requests_seconds_count{method=\"PUT\",uri=\"/hos/{bucket}/**\",project=\"$Project\"}[2m]))", + "legend":"Total", + "name":"A", + "orderNum":0, + "id":108727, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(irate(http_server_requests_seconds_count{method=\"PUT\",status=~\"2..\",uri=\"/hos/{bucket}/**\",project=\"$Project\"}[2m]))", + "legend":"Success", + "name":"B", + "orderNum":1, + "id":108728, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(irate(http_server_requests_seconds_count{method=\"PUT\",status!~\"2..\",uri=\"/hos/{bucket}/**\",project=\"$Project\"}[2m]))", + "legend":"Failure", + "name":"C", + "orderNum":2, + "id":108729, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Report Success", + "span":"3.0", + "height":"1.0", + "type":"Stat", + "unit":"short", + "weight":3, + "param":{ + "comparison":"none", + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "colorMode":"value", + "thresholds":[], + "sparklineMode":"line", + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":false + }, + "valueMapping":[ + { + "color":{ + "bac":"#E81616", + "text":"#000" + }, + "display":"{{A.$value}}", + "show":false, + "text":"", + "type":"value", + "value":0 + } + ], + "dataLink":[], + "nullType":"connected", + "statistics":"last" + }, + "remark":"Total number of report jobs successfully executed", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(report_success_count_total{module=~\".*Saved-query-scheduler\",project=\"$Project\"})", + "legend":"Report Success", + "name":"A", + "orderNum":0, + "id":108730, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Report Failure", + "span":"3.0", + "height":"1.0", + "type":"Stat", + "unit":"none", + "weight":4, + "param":{ + "comparison":"none", + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "colorMode":"value", + "thresholds":[], + "sparklineMode":"line", + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "valueMapping":[ + { + "regx":">10", + "color":{ + "bac":"#F78539FF", + "text":"#000" + }, + "display":"{{value}}", + "show":true, + "from":1, + "text":"", + "to":100, + "type":"range", + "error":false, + "value":10 + }, + { + "color":{ + "bac":"#E30521FF", + "text":"#000000FF" + }, + "display":"{{value}}", + "show":true, + "from":100, + "to":10000, + "type":"range", + "error":false + } + ], + "text":"value", + "dataLink":[], + "nullType":"connected", + "statistics":"last" + }, + "remark":"Total number of report jobs failed", + "x":"3.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(report_fail_count_total{module=~\".*Saved-query-scheduler\",project=\"$Project\"})", + "legend":"Report fail sum", + "name":"A", + "orderNum":0, + "id":108731, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Success(today)", + "span":"3.0", + "height":"1.0", + "type":"Stat", + "unit":"short", + "weight":5, + "param":{ + "comparison":"none", + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "colorMode":"value", + "thresholds":[], + "sparklineMode":"line", + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":false + }, + "valueMapping":[ + { + "color":{ + "bac":"#fff", + "text":"#000" + }, + "display":"{{A.$value}}", + "show":false, + "text":"", + "type":"value", + "value":0 + } + ], + "dataLink":[], + "nullType":"connected", + "statistics":"last" + }, + "remark":"The total number of jobs successfully executed today", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(triggerCountSucTotal{module=~\".*Galaxy-job-admin\",project=\"$Project\"}) / sum(up{module=~\".*Galaxy-job-admin\",project=\"$Project\"})", + "legend":"Job Success", + "name":"A", + "orderNum":0, + "id":108732, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Failure(today)", + "span":"3.0", + "height":"1.0", + "type":"Stat", + "unit":"short", + "weight":6, + "param":{ + "comparison":"none", + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "colorMode":"value", + "thresholds":[], + "sparklineMode":"line", + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "valueMapping":[ + { + "color":{ + "bac":"#FF9C59FF", + "icon":"#FF9830FF", + "text":"#000" + }, + "display":"{{value}}", + "show":true, + "from":1, + "text":"", + "to":50, + "type":"range", + "error":false, + "value":0 + }, + { + "color":{ + "bac":"#E30521FF", + "icon":"#df2fefFF", + "text":"#000000FF" + }, + "display":"{{value}}", + "show":true, + "from":51, + "to":10000, + "type":"range", + "error":false, + "value":50 + } + ], + "text":"value", + "dataLink":[], + "nullType":"connected", + "statistics":"last" + }, + "remark":"The total number of jobs failed executed today", + "x":"9.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(triggerCountFailTotal{module=~\".*Galaxy-job-admin\",project=\"$Project\"}) / sum(up{module=~\".*Galaxy-job-admin\",project=\"$Project\"})", + "legend":"Job Failure", + "name":"A", + "orderNum":0, + "id":108733, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Galaxy-gateway-nginx", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":7, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Project", + "varValue":"TSG-DT", + "operator":"notEqual" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"1.37", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Server Requests (asset,code)", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":8, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The requests counter per second of each Gateway Nginx asset.", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(irate(nginx_vts_server_requests_total{module=~\".*Galaxy-gateway-nginx\",code!=\"total\",project=\"$Project\"}[5m])) by (code,asset)", + "legend":"{{asset}}_[{{code}}]", + "name":"A", + "orderNum":0, + "id":108735, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Upstream Requests (asset,code)", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":9, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The upstream requests the counter rate of each Gateway Nginx asset.", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(irate(nginx_vts_upstream_requests_total{module=~\".*Galaxy-gateway-nginx\",code!=\"total\",project=\"$Project\"}[5m])) by (code,asset)", + "legend":"{{asset}}_[{{code}}]", + "name":"A", + "orderNum":0, + "id":108736, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Upstream Response Time (backend,upstream)", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"seconds(s)", + "weight":10, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The only upstream response processing time of each endpoint", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"max(irate(nginx_vts_upstream_response_seconds_total{module=~\".*Galaxy-gateway-nginx\",project=\"$Project\"}[5m])) by (backend, upstream)", + "legend":"{{backend}}-{{upstream}}", + "name":"A", + "orderNum":0, + "id":108737, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Connections", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"none", + "weight":11, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#ed0937", + "id":"4175dd3a" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of connections for the nginx proxy in each asset", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"nginx_vts_main_connections{module=~\".*Galaxy-gateway-nginx\",status=\"active\",project=\"$Project\"}", + "legend":"{{asset}}-{{status}}", + "name":"A", + "orderNum":0, + "id":108738, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Galaxy-qgw-service", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":12, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Project", + "varValue":"TSG-DT", + "operator":"notEqual" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"1.74", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Http Error Responses", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":13, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#ca8bf9", + "id":"84abc9c" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The number of failed requests for QGW .", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"count(http_server_requests_seconds_count{module=~\".*Galaxy-qgw-service\", status!~\"2..\",uri!~\".*/prometheus\",project=\"$Project\"})", + "legend":"count", + "name":"A", + "orderNum":0, + "id":108740, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Http Slow Query", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":14, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "val":"300", + "color":"#d64f40", + "id":"32e399eb" + } + ], + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The number of slow queries", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"count(http_server_requests_seconds_max{module=~\".*Galaxy-qgw-service\",uri!~\".*/prometheus\",project=\"$Project\"}>90)", + "legend":">90s", + "name":"A", + "orderNum":0, + "id":108741, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Memory Usage", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":15, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"connected" + }, + "remark":"The memory usage in each asset", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(asset) (jvm_memory_used_bytes{module=~\".*Galaxy-qgw-service\",project=\"$Project\"}) / sum by(asset) (jvm_memory_max_bytes{module=~\".*Galaxy-qgw-service\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108742, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Saved-query-scheduler", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":16, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Project", + "varValue":"TSG-DT", + "operator":"notEqual" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"2.1", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Report Success / All", + "span":"2.0", + "height":"2.0", + "type":"Stat", + "unit":"percent(0-100)", + "weight":17, + "param":{ + "comparison":"none", + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "colorMode":"value", + "thresholds":[], + "sparklineMode":"line", + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "valueMapping":[ + { + "color":{ + "bac":"#fff", + "text":"#000" + }, + "display":"{{A.$value}}", + "show":false, + "text":"", + "type":"value", + "value":0 + } + ], + "dataLink":[], + "nullType":"connected", + "statistics":"last" + }, + "remark":"The success rate of Report jobs", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(report_success_count_total{module=~\".*Saved-query-scheduler\",project=\"$Project\"}) / (sum(report_success_count_total{module=~\".*Saved-query-scheduler\",project=\"$Project\"}) + sum(report_fail_count_total{module=~\".*Saved-query-scheduler\",project=\"$Project\"})) * 100", + "name":"A", + "orderNum":0, + "id":108746, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Memory Usage", + "span":"5.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":18, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The memory usage in each asset", + "x":"2.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(asset) (jvm_memory_used_bytes{module=~\".*Saved-query-scheduler\",project=\"$Project\"}) / sum by(asset) (jvm_memory_max_bytes{module=~\".*Saved-query-scheduler\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108747, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"CPU Usage", + "span":"5.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":19, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The CPU usage in each asset", + "x":"7.0", + "y":"0.0", + "elements":[ + { + "expression":"system_cpu_usage{module=~\".*Saved-query-scheduler\",project=\"$Project\"}", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108748, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Galaxy-hos-service", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":20, + "param":{ + "thresholds":[], + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"10.84", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Put Object Requests/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":21, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The number of put objects per second", + "x":"0.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by (asset)(irate(http_server_requests_seconds_count{method=\"PUT\",uri=\"/hos/{bucket}/**\",project=\"$Project\"}[5m]))", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108750, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Error Requests/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":22, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The number of failed requests per second", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(status)(irate(http_server_requests_seconds_count{uri=~\"/hos/.*\",status!~\"2..\",project=\"$Project\"}[5m]))", + "legend":"{{status}}", + "name":"A", + "orderNum":0, + "id":108751, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Download File Requests/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":23, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The number of Download File requests per second", + "x":"6.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by (asset)(irate(http_server_requests_seconds_count{method=\"GET\",uri=\"/hos/{bucket}/**\",project=\"$Project\"}[5m]))", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108752, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Write Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":24, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Collecting and processing files with incoming byte rate in each asset", + "x":"6.0", + "y":"6.0", + "elements":[ + { + "expression":"irate(dashInfo{severity=\"fileBytes\",module=~\".*Galaxy-hos-service\",project=\"$Project\"}[5m])", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108753, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Put Requests By Size Files/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":25, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The number of put requests per second with size distribution", + "x":"0.0", + "y":"10.0", + "elements":[ + { + "expression":"sum(irate(dashInfo{severity=\"smallFileCount\",module=~\".*Galaxy-hos-service\",project=\"$Project\"}[5m]))", + "legend":"<1MB", + "name":"A", + "orderNum":0, + "id":108754, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(irate(dashInfo{severity=\"middleFileCount\",module=\"Galaxy-hos-service\",project=\"$Project\"}[5m]))", + "legend":"1-10MB", + "name":"B", + "orderNum":1, + "id":108755, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(irate(dashInfo{severity=\"bigFileCount\",module=\"Galaxy-hos-service\",project=\"$Project\"}[5m]))", + "legend":">10MB", + "name":"C", + "orderNum":2, + "id":108756, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Memory Usage", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":26, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The memory usage in each asset", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(asset)(jvm_memory_used_bytes{module=~\".*Galaxy-hos-service\",project=\"$Project\"}>0) / sum by(asset) (jvm_memory_max_bytes{module=~\".*Galaxy-hos-service\",project=\"$Project\"}>0)", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108757, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"CPU Usage", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":27, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The CPU usage in each asset", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"process_cpu_usage{module=~\".*Galaxy-hos-service\",project=\"$Project\"}", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108758, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"HOS Node Requests/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":28, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "total", + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The number of hos requests per second", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(irate(http_server_requests_seconds_count{uri=\"/hos/{bucket}/**\",project=\"$Project\"}[5m])) by (asset)", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108759, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"HOS Put Files/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":29, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The number of files uploaded per second", + "x":"0.0", + "y":"6.0", + "elements":[ + { + "expression":"sum by(datacenter)(irate(dashInfo{severity=\"fileCount\",module=~\".*Galaxy-hos-service\",project=\"$Project\"}[5m]))", + "legend":"{{datacenter}}", + "name":"A", + "orderNum":0, + "id":108760, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"HOS TTL Delete Files/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":30, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The number of ttl deleted files per second.", + "x":"6.0", + "y":"10.0", + "elements":[ + { + "expression":"sum by(datacenter)(irate(dashInfo{severity=\"ttlDeleteFileCount\",module=~\".*Galaxy-hos-service\",project=\"$Project\"}[5m]))", + "legend":"DeleteCount-{{datacenter}}", + "name":"A", + "orderNum":0, + "id":108761, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(datacenter)(irate(dashInfo{severity=\"ttlDeleteFileErrorCount\",module=\"Galaxy-hos-service\",project=\"$Project\"}[5m]))", + "legend":"DeleteErrorCount-{{datacenter}}", + "name":"B", + "orderNum":1, + "id":108762, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"HOS Put File By Bucket/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":31, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The number of files for bucket uploaded per second", + "x":"0.0", + "y":"8.0", + "elements":[ + { + "expression":"sum by(parent)(irate(Hadoop_HBase_metric_putCount{namespace=\"default\",type=\"index\",indextype=\"time\",project=\"$Project\"}[5m]))", + "legend":"{{parent}}", + "name":"A", + "orderNum":0, + "id":108763, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"HOS Put Requests By Bucket/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":32, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"The number of put requests for bucket per second", + "x":"6.0", + "y":"8.0", + "elements":[ + { + "expression":"sum by(table)(irate(Hadoop_HBase_metric_putCount{namespace=\"default\",type=\"data\",project=\"$Project\"}[5m]))", + "legend":"{{table}}", + "name":"A", + "orderNum":0, + "id":108764, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Galaxy-job-service", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":33, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"Project", + "varValue":"TSG-DT", + "operator":"notEqual" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":true, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"2.47", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Job Success/All", + "span":"2.0", + "height":"2.0", + "type":"Stat", + "unit":"percent(0.0-1.0)", + "weight":34, + "param":{ + "comparison":"none", + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "colorMode":"value", + "thresholds":[], + "sparklineMode":"line", + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":false + }, + "valueMapping":[ + { + "color":{ + "bac":"#fff", + "text":"#000" + }, + "display":"{{A.$value}}", + "show":false, + "text":"", + "type":"value", + "value":0 + } + ], + "dataLink":[], + "nullType":"connected", + "statistics":"last" + }, + "remark":"The success rate of jobs", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(jobLogSuccessCount{module=~\".*Galaxy-job-admin\",project=\"$Project\"})/sum(jobLogCount{module=~\".*Galaxy-job-admin\",project=\"$Project\"})", + "name":"A", + "orderNum":0, + "id":108765, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Executor Sum", + "span":"2.0", + "height":"2.0", + "type":"Stat", + "unit":"short", + "weight":35, + "param":{ + "comparison":"none", + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "colorMode":"value", + "thresholds":[], + "sparklineMode":"line", + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "valueMapping":true + }, + "valueMapping":[ + { + "color":{ + "bac":"#fff", + "text":"#000" + }, + "display":"{{A.$value}}", + "show":false, + "text":"", + "type":"value", + "value":0 + } + ], + "dataLink":[], + "nullType":"connected", + "statistics":"last" + }, + "remark":"The total number of executed jobs", + "x":"2.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(jobLogCount{module=~\".*Galaxy-job-admin\",project=\"$Project\"})/count(jobLogCount{module=~\".*Galaxy-job-admin\",project=\"$Project\"})", + "name":"A", + "orderNum":0, + "id":108766, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Admin Response Latency", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"seconds(s)", + "weight":36, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The response time of admin service", + "x":"4.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(rate(http_server_requests_seconds_sum{module=~\".*Galaxy-job-admin\", status!~\"5..\",project=\"$Project\"}[5m]))/sum(rate(http_server_requests_seconds_count{module=~\".*Galaxy-job-executor\", status!~\"5..\",project=\"$Project\"}[5m]))", + "legend":"AVG", + "name":"A", + "orderNum":0, + "id":108772, + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"max(http_server_requests_seconds_max{module=\"Galaxy-job-admin\", status!~\"5..\",project=\"$Project\"})", + "legend":"MAX", + "name":"B", + "orderNum":1, + "id":108773, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Admin Memory Usage", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":37, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The memory usage in each asset", + "x":"0.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by(asset) (jvm_memory_used_bytes{module=~\".*Galaxy-job-admin\",project=\"$Project\"}) / sum by(asset) (jvm_memory_max_bytes{module=~\".*Galaxy-job-admin\",project=\"$Project\"})", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108774, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Admin Error Requests/sec", + "span":"4.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":38, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"connected" + }, + "remark":"The number of failed requests per second", + "x":"8.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(rate(http_server_requests_seconds_count{module=~\".*Galaxy-job-admin\", status!~\"2..\",project=\"$Project\"}[5m]))", + "legend":"used_{{asset}}", + "name":"A", + "orderNum":0, + "id":108775, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Admin CPU Usage", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":39, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"connected" + }, + "remark":"The CPU usage in each asset", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"system_cpu_usage{module=~\".*Galaxy-job-admin\",project=\"$Project\"}", + "legend":"{{asset}}_system_cpu_usage", + "name":"A", + "orderNum":0, + "id":108776, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Executor Memory Used", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes(IEC)", + "weight":40, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"connected" + }, + "remark":"The memory usage in each asset", + "x":"0.0", + "y":"6.0", + "elements":[ + { + "expression":"sum(jvm_memory_used_bytes{module=~\".*Galaxy-job-executor\",project=\"$Project\"})by(asset)", + "legend":"used_{{asset}}", + "name":"A", + "orderNum":0, + "id":108778, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Executor Error Log", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":41, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"connected" + }, + "remark":"The number of failed logs per second in each asset", + "x":"6.0", + "y":"4.0", + "elements":[ + { + "expression":"increase(logback_events_total{module=~\".*Galaxy-job-executor\",level=\"error\",project=\"$Project\"} [5m])", + "legend":" {{asset}}", + "name":"A", + "orderNum":0, + "id":108779, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Executor CPU Usage", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0.0-1.0)", + "weight":42, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[ + "avg", + "last", + "max" + ], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The CPU usage in each asset", + "x":"6.0", + "y":"6.0", + "elements":[ + { + "expression":"system_cpu_usage{module=~\".*Galaxy-job-executor\",project=\"$Project\"}", + "legend":"{{asset}}", + "name":"A", + "orderNum":0, + "id":108780, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Delete Old Log", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":43, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "avg" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#18ba46", + "id":"81efdaf" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"When the disk is fully loaded, the deletion log is triggered", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by (log_type) (delete_old_log_total{module=\"Galaxy-job-executor\"})", + "legend":"{{log_type}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Galaxy-hos-nginx", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":44, + "param":{ + "thresholds":[], + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":false, + "valueMapping":false + }, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true, + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"23.21", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Connections", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"none", + "weight":45, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[ + { + "color":"#ed0937", + "id":"94c2b08" + } + ], + "thresholdShow":true, + "enable":{ + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of connections for the nginx proxy in each asset", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"nginx_vts_main_connections{module=~\".*Galaxy-hos-nginx\",status=\"active\",project=\"$Project\"}", + "legend":"{{asset}}-{{status}}", + "name":"A", + "orderNum":0, + "id":108782, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Server Requests (asset,code)", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":46, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The requests counter per second of each HOS Nginx asset.", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(irate(nginx_vts_server_requests_total{module=~\".*Galaxy-hos-nginx\",code!=\"total\",project=\"$Project\"}[5m])) by (code,asset)", + "legend":"{{asset}}_[{{code}}]", + "name":"A", + "orderNum":0, + "id":108783, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Upstream Requests (asset,code)", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":47, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The upstream requests the counter rate of each HOS Nginx asset.", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(irate(nginx_vts_upstream_requests_total{module=~\".*Galaxy-hos-nginx\",code!=\"total\",project=\"$Project\"}[5m])) by (code,asset)", + "legend":"{{asset}}_[{{code}}]", + "name":"A", + "orderNum":0, + "id":108784, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Upstream Response Time (backend,upstream)", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"seconds(s)", + "weight":48, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":false, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "legend":{ + "values":[], + "placement":"bottom" + }, + "link":"", + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "thresholds":[], + "enable":{ + "thresholds":true, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":true + }, + "style":"line", + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The only upstream response processing time of each endpoint", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"max(irate(nginx_vts_upstream_response_seconds_total{module=~\".*Galaxy-hos-nginx\",project=\"$Project\"}[5m])) by (backend, upstream)", + "legend":"{{backend}}-{{upstream}}", + "name":"A", + "orderNum":0, + "id":108785, + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + } + ], + "children":[] + }, + { + "name":"[24.09] Data Transporter Data Flow Template", + "type":"template", + "varType":0, + "param":{ + "chartShare":"none", + "variables":[ + { + "regex":"", + "expression":"", + "customOptions":"DataCenter1,DataCenter2", + "show":true, + "name":"DataCenter", + "allOption":false, + "type":"custom", + "multi":false + } + ], + "report":{ + "schedule":{ + "repeat":1, + "etime":"", + "stime":"", + "type":2, + "nums":[] + }, + "receivers":[], + "enable":false, + "range":{ + "unit":"day", + "interval":1, + "type":"previous" + } + }, + "refresh":0, + "defaultTimeRange":4 + }, + "remark":"The template for TSG Data Transporter DataFlow at multi-datacenter deployment mode.", + "charts":[ + { + "name":"Summary", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":0, + "param":{ + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "collapse":false + }, + "remark":"", + "x":"0.0", + "y":"1.38", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Log Ingestion Rate Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":1, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "link":"", + "thresholds":[ + { + "color":"#6339a3", + "id":"7715ee9" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRANSACTION-RECORD|SESSION-RECORD|BGP-RECORD|PROXY-EVENT\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"Total", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRANSACTION-RECORD\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"TRANSACTION-RECORD", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"SESSION-RECORD\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"SESSION-RECORD", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"BGP-RECORD\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"BGP-RECORD", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"PROXY-EVENT\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"PROXY-EVENT", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Metric Ingestion Rate Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":2, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "link":"", + "thresholds":[ + { + "color":"#b6a6ed", + "id":"6f07e8bf" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"DOS-SKETCH-RECORD|NETWORK-TRAFFIC-METRIC|OBJECT-STATISTICS-METRIC|POLICY-RULE-METRIC|STATISTICS-RULE-METRIC|SYS-STORAGE|TRAFFIC-TOP-METRIC|VOIP-CONVERSATION-RECORD|TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"Total", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"DOS-SKETCH-RECORD\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"DOS-SKETCH-RECORD", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"NETWORK-TRAFFIC-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"NETWORK-TRAFFIC-METRIC", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"OBJECT-STATISTICS-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"OBJECT-STATISTICS-METRIC", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"POLICY-RULE-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"POLICY-RULE-METRIC", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"STATISTICS-RULE-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"STATISTICS-RULE-METRIC", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"SYS-STORAGE\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"SYS-STORAGE", + "name":"G", + "orderNum":6, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRAFFIC-TOP-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"TRAFFIC-TOP-METRIC", + "name":"H", + "orderNum":7, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"VOIP-CONVERSATION-RECORD\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"VOIP-CONVERSATION-RECORD", + "name":"I", + "orderNum":8, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"TRAFFIC-TOP-DESTINATION-IP-METRIC", + "name":"J", + "orderNum":9, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"File Chunk Ingestion Rate Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":3, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "link":"", + "thresholds":[ + { + "color":"#4ff9a7", + "id":"b755935" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\".*FILE-STREAM-RECORD\",datacenter=\"$DataCenter\"})", + "legend":"Total", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (topic)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\".*FILE-STREAM-RECORD\",datacenter=\"$DataCenter\"})", + "legend":"{{topic}}", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Traffic IN/OUT", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":4, + "param":{ + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "collapse":false + }, + "remark":"", + "x":"0.0", + "y":"1.76", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Log Ingestion IN/OUT Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":5, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#ce8c08", + "id":"58d58a08" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesInPerSec\",topic=~\"TRANSACTION-RECORD|SESSION-RECORD|BGP-RECORD|PROXY-EVENT\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesOutPerSec\",topic=~\"TRANSACTION-RECORD|SESSION-RECORD|BGP-RECORD|PROXY-EVENT\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Metric Ingestion IN/OUT Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":6, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#ce8c08", + "id":"e62d783" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesInPerSec\",topic=~\"DOS-SKETCH-RECORD|NETWORK-TRAFFIC-METRIC|OBJECT-STATISTICS-METRIC|POLICY-RULE-METRIC|STATISTICS-RULE-METRIC|SYS-STORAGE|TRAFFIC-TOP-METRIC|VOIP-CONVERSATION-RECORD|TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesOutPerSec\",topic=~\"DOS-SKETCH-RECORD|NETWORK-TRAFFIC-METRIC|OBJECT-STATISTICS-METRIC|POLICY-RULE-METRIC|STATISTICS-RULE-METRIC|SYS-STORAGE|TRAFFIC-TOP-METRIC|VOIP-CONVERSATION-RECORD|TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"File Chunk Ingestion IN/OUT Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":7, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#ce8c08", + "id":"1752506" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesInPerSec\",topic=~\".*FILE-STREAM-RECORD\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesOutPerSec\",topic=~\".*FILE-STREAM-RECORD\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Session Records", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":8, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true + }, + "remark":"", + "x":"0.0", + "y":"2.15", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"ETL Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":9, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#2ce059", + "id":"22e472c9" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"SESSION-RECORD\",datacenter=\"$DataCenter\"})", + "legend":"Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"SESSION-RECORD-PROCESSED\",datacenter=\"$DataCenter\"})", + "legend":"Transform", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n ) or vector(0) )+ sum(rate( flink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n ) or vector(0)\n)", + "legend":"Sink", + "name":"D", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Source-Sink Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":10, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"fb7c1ff" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Dropped", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Error", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Processor Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":11, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"ce0253" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Dropped", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Error", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_events{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Source-Sink Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":12, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"716d3872" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_bytes{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_bytes{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_bytes{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Dropped", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_bytes{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Error", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_bytes{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_bytes{job_name=\"etl_session_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Transaction Records", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":13, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true + }, + "remark":"", + "x":"0.0", + "y":"2.53", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Source-Sink Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":14, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"7efeb2" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_bytes{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_bytes{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_bytes{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Dropped", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_bytes{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Error", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_bytes{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_bytes{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"ETL Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":15, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#ed0eaa", + "id":"2027e85c" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"TRANSACTION-RECORD\",datacenter=\"$DataCenter\"})", + "legend":"Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"TRANSACTION-RECORD-PROCESSED\",datacenter=\"$DataCenter\"})", + "legend":"Transform", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n ) or vector(0) )+ sum(rate( flink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n ) or vector(0)\n)", + "legend":"Sink", + "name":"D", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Source-Sink Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":16, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"3d8851d6" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Dropped", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Error", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Processor Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":17, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"3445b88" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Dropped", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Error", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_events{job_name=\"etl_transaction_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Proxy Events", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":18, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true + }, + "remark":"", + "x":"0.0", + "y":"2.91", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"ETL Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":19, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#2a16ff", + "id":"7dd05d48" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"PROXY-EVENT\",datacenter=\"$DataCenter\"})", + "legend":"Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"PROXY-EVENT-PROCESSED\",datacenter=\"$DataCenter\"})", + "legend":"Transform", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n ) or vector(0)) + sum(rate( flink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n ) or vector(0)\n)", + "legend":"Sink", + "name":"D", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Source-Sink Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":20, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"6fbdba84" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Dropped", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Error", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DT\"}[5m]\n )\n)", + "legend":"Sink-Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DT\"}[5m]\n )\n)", + "legend":"Sink-Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Source-Sink Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":21, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"1692816" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_bytes{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_bytes{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_bytes{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Dropped", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_bytes{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Error", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_bytes{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_bytes{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Processor Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":22, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"27e6e467" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Dropped", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Error", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_events{job_name=\"etl_proxy_event_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"DoS Events", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":23, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true + }, + "remark":"", + "x":"0.0", + "y":"3.29", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"ETL Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":24, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#6936aa", + "id":"1ed16a0e" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"DOS-SKETCH-RECORD\",datacenter=\"$DataCenter\"})", + "legend":"Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"DOS-EVENT\",datacenter=\"$DataCenter\"})", + "legend":"Transform", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n ) or vector(0)) + sum(rate( flink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n ) or vector(0)\n)", + "legend":"Sink", + "name":"D", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Source-Sink Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":25, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"dd3cf51" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Dropped", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Error", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Source-Sink Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":26, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"6881cb13" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_bytes{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_bytes{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_bytes{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Dropped", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_bytes{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Error", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_bytes{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_bytes{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Processor Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":27, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"667b36dc" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Dropped", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Error", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_events{job_name=\"dos_sketch_record_kafka_to_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"VoIP Records", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":28, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true + }, + "remark":"", + "x":"0.0", + "y":"3.67", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"ETL Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":29, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#c02bdb", + "id":"6e96ebc8" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Collecting and processing VoIP Records per second", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"VOIP-RECORD\",datacenter=\"$DataCenter\"}) ", + "legend":"Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"VOIP-RECORD-PROCESSED\",datacenter=\"$DataCenter\"})", + "legend":"Transform", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"VOIP-CONVERSATION-RECORD\",datacenter=\"$DataCenter\"})", + "legend":"Transform(Connection）", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n ) or vector(0) )+ sum(rate( flink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n ) or vector(0)\n)", + "legend":"Sink", + "name":"E", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Source-Sink Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":30, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"2b063b85" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Dropped", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Error", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Source-Sink Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":31, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"11c1730" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_bytes{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_bytes{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_bytes{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Dropped", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_bytes{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Source:_kafka_source\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Source-Error", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_bytes{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_bytes{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"Sink:_kafka_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink-Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Processor Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":32, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#70c8ea", + "id":"2324386" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Dropped", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Error", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Out", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_in_events{job_name=\"voip_record_kafka_to_ndc_kafka\",operator_name=\"etl_processor\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"In", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"TRAFFIC-FILE-STREAM-RECORD", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":33, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true + }, + "remark":"", + "x":"0.0", + "y":"4.05", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Traffic File Stream Record Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":34, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "min", + "max", + "avg" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"e67313c" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of records written to kafka per second / Number of records consumed per second / Kafka lag", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\".*FILE-STREAM-RECORD\",datacenter=\"$DataCenter\"})", + "legend":"Total-Ingestion", + "name":"D", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(topic)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\".*FILE-STREAM-RECORD\",datacenter=\"$DataCenter\"})", + "legend":"{{topic}}-Ingestion", + "name":"A", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_KafkaConsumer_records_consumed_rate{job_name=~\".*file_chunk_combiner\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-Consumer", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job_name=~\".*file_chunk_combiner\",datacenter=\"$DataCenter\"} >= 0)", + "legend":"{{job_name}}-Lag", + "name":"B", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"File Chunk IN/OUT Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":35, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "min", + "max", + "avg" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"e46fab5" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Bytes of chunk input(source) and output(sink) per second.", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBytesInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBytesOutPerSecond{file_chunk_combiner=~\"sink_.*\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"File Chunk Combiner IN/OUT Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":36, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "min", + "max", + "avg" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"4b692fe3" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of chunk inputs and outputs per second of the window combiner.", + "x":"0.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numChunksInPerSecond{file_chunk_combiner=\"window_combine_chunk\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numChunksOutPerSecond{file_chunk_combiner=\"window_combine_chunk\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"File Chunk Error Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":37, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#ed5cea", + "id":"337035b" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of error chunks per second of the operator.", + "x":"0.0", + "y":"6.0", + "elements":[ + { + "expression":"sum by(job_name,file_chunk_combiner)(flink_taskmanager_job_task_operator_file_chunk_combiner_numErrorChunksPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-{{file_chunk_combiner}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"File Chunk Dropped Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":38, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#5b3bf9", + "id":"5655796a" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of chunk drops per second of the operator.", + "x":"6.0", + "y":"6.0", + "elements":[ + { + "expression":"sum by(job_name,file_chunk_combiner)(flink_taskmanager_job_task_operator_file_chunk_combiner_numChunksFilterPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-{{file_chunk_combiner}}-Filter", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name,file_chunk_combiner)(flink_taskmanager_job_task_operator_file_chunk_combiner_numChunksRateLimitDropPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-{{file_chunk_combiner}}-RateLimitDrop", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numDuplicateChunksInPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-Duplicate", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Ingestion Chunk Distribution By Size Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":39, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#f90ede", + "id":"5960455e" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of chunk inputs(source) per second by size.", + "x":"6.0", + "y":"12.0", + "elements":[ + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numLessThan1KBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-0-1K", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween1KBAnd3KBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-1-3K", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween3KBAnd5KBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-3-5K", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween5KBAnd10KBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-5-10K", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween10KBAnd50KBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-10-50K", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween50KBAnd100KBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-50-100K", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numGreaterThan100KBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-100K-∞", + "name":"G", + "orderNum":6, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Restarts", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":40, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"665680f" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of task restarts per second.", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"max by(job_name) ((flink_jobmanager_job_numRestarts{job_name=~\".*file_chunk_combiner$\",datacenter=\"$DataCenter\"}) - (flink_jobmanager_job_numRestarts{job_name=~\".*file_chunk_combiner$\",datacenter=\"$DataCenter\"} offset 30s))", + "legend":"{{job_name}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job Memory Used", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"percent(0-100)", + "weight":41, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"57f73fd3" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"18.0", + "elements":[ + { + "expression":"(flink_taskmanager_Status_JVM_Memory_Heap_Used{exported_job=~\".*file_chunk_combiner.*\",datacenter=\"$DataCenter\"} / flink_taskmanager_Status_JVM_Memory_Heap_Max{exported_job=~\".*file_chunk_combiner.*\",datacenter=\"$DataCenter\"}) * 100", + "legend":"{{asset}}-HeapUsed", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Job GC Metrics", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":42, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"fb65a89" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"18.0", + "elements":[ + { + "expression":"flink_taskmanager_Status_JVM_GarbageCollector_G1_Old_Generation_Count{exported_job=~\".*file_chunk_combiner.*\",datacenter=\"$DataCenter\"}", + "legend":"{{asset}}-G1OldCount", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"flink_taskmanager_Status_JVM_GarbageCollector_G1_Old_Generation_Time{exported_job=~\".*file_chunk_combiner.*\",datacenter=\"$DataCenter\"}", + "legend":"{{asset}}-G1OldTime", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"flink_taskmanager_Status_JVM_GarbageCollector_G1_Young_Generation_Count{exported_job=~\".*file_chunk_combiner.*\",datacenter=\"$DataCenter\"}", + "legend":"{{asset}}-G1YoungCount", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"flink_taskmanager_Status_JVM_GarbageCollector_G1_Young_Generation_Time{exported_job=~\".*file_chunk_combiner.*\",datacenter=\"$DataCenter\"}", + "legend":"{{asset}}-G1YoungTime", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"File Chunk IN/OUT Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":43, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "min", + "max", + "avg" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"be25170" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of chunk inputs(source) and outputs(sink) per second.", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numChunksOutPerSecond{file_chunk_combiner=~\"sink_.*\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"File IN/OUT Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":44, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "min", + "max", + "avg" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"42a0f6a" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of file inputs(source) and outputs(sink) per second.", + "x":"6.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numStartChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numFilesOutPerSecond{file_chunk_combiner=~\"sink_.*\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Sink Chunk Distribution By Size Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":45, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#f90ede", + "id":"4337ac0c" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of chunk outputs(sink) per second by size.", + "x":"6.0", + "y":"14.0", + "elements":[ + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numLessThan1KBChunksOutPerSecond{file_chunk_combiner=~\"sink_.*\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-0-1K", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween1KBAnd5KBChunksOutPerSecond{file_chunk_combiner=~\"sink_.*\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-1-5K", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween5KBAnd10KBChunksOutPerSecond{file_chunk_combiner=~\"sink_.*\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-5-10K", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween10KBAnd100KBChunksOutPerSecond{file_chunk_combiner=~\"sink_.*\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-10-100K", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween100KBAnd1MBChunksOutPerSecond{file_chunk_combiner=~\"sink_.*\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-100K-1M", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numGreaterThan1MBChunksOutPerSecond{file_chunk_combiner=~\"sink_.*\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-1M-∞", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Ingestion Chunk Distribution By Offset Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":46, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"3486b0ae" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of chunk inputs(source) per second by offset.", + "x":"0.0", + "y":"14.0", + "elements":[ + { + "expression":"sum by (job_name) (flink_taskmanager_job_task_operator_file_chunk_combiner_numOffsetLessThan10KBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}_0-10K", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (job_name) (flink_taskmanager_job_task_operator_file_chunk_combiner_numOffsetBetween10KBAnd100KBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}_10-100K", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (job_name) (flink_taskmanager_job_task_operator_file_chunk_combiner_numOffsetBetween100KBAnd1MBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}_100K-1M", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (job_name) (flink_taskmanager_job_task_operator_file_chunk_combiner_numOffsetBetween1MBAnd10MBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}_1-10M", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (job_name) (flink_taskmanager_job_task_operator_file_chunk_combiner_numOffsetBetween10MBAnd100MBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}_10-100M", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (job_name) (flink_taskmanager_job_task_operator_file_chunk_combiner_numOffsetBetween100MBAnd1GBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}_100M-1G", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (job_name) (flink_taskmanager_job_task_operator_file_chunk_combiner_numOffsetGreaterThan1GBChunksInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}_1G-∞", + "name":"G", + "orderNum":6, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Null File IN/OUT Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":47, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"beaef36" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Number of null file inputs and outputs per second.", + "x":"6.0", + "y":"16.0", + "elements":[ + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numNullFilesInPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numNullFilesOutPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-Out", + "name":"D", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Other File Chunk IN/OUT Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":48, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"199c9f74" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"16.0", + "elements":[ + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numStartChunksInPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-StartChunks-In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numEndChunksInPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-EndChunks-In", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numStartChunksOutPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-StartChunks-Out", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numEndChunksOutPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-EndChunks-Out", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numCompleteFilesOutPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-CompleteFiles-Out", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numCompleteEmlFilesOutPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-CompleteEmlFiles-Out", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numCompleteTxtFilesOutPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-CompleteTxtFiles-Out", + "name":"G", + "orderNum":6, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numChunksDelayPerSecond{datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-DelayChunks-In", + "name":"I", + "orderNum":7, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Ingestion File Distribution By Size Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":53, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#42a6d1", + "id":"bc8100f" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"Number of file inputs(source) per second by size.", + "x":"0.0", + "y":"12.0", + "elements":[ + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numLessThan10KBFilesInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-0-10K", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween10KBAnd100KBFilesInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-10-100K", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween100KBAnd1MBFilesInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-100K-1M", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween1MBAnd10MBFilesInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-1-10M", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween10MBAnd100MBFilesInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-10-100M", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numBetween100MBAnd1GBFilesInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-100M-1G", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(job_name)(flink_taskmanager_job_task_operator_file_chunk_combiner_numGreaterThan1GBFilesInPerSecond{file_chunk_combiner=\"map_message_pack\",datacenter=\"$DataCenter\"})", + "legend":"{{job_name}}-1G-∞", + "name":"G", + "orderNum":6, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Overview", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":54, + "param":{ + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "collapse":false + }, + "remark":"", + "x":"0.0", + "y":"1.0", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"DataCenter Metric Ingestion IN/OUT Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":55, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "link":"", + "thresholds":[ + { + "color":"#ce8c08", + "id":"75eb411" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by(datacenter)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesInPerSec\",topic=~\"DOS-SKETCH-RECORD|NETWORK-TRAFFIC-METRIC|OBJECT-STATISTICS-METRIC|POLICY-RULE-METRIC|STATISTICS-RULE-METRIC|SYS-STORAGE|TRAFFIC-TOP-METRIC|VOIP-CONVERSATION-RECORD|TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\"})", + "legend":"{{datacenter}}-In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(datacenter)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesOutPerSec\",topic=~\"DOS-SKETCH-RECORD|NETWORK-TRAFFIC-METRIC|OBJECT-STATISTICS-METRIC|POLICY-RULE-METRIC|STATISTICS-RULE-METRIC|SYS-STORAGE|TRAFFIC-TOP-METRIC|VOIP-CONVERSATION-RECORD|TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\"})", + "legend":"{{datacenter}}-Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"DataCenter Log Ingestion Rate Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":56, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "link":"", + "thresholds":[ + { + "color":"#4740ce", + "id":"a2df341" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by(datacenter)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRANSACTION-RECORD|SESSION-RECORD|BGP-RECORD|PROXY-EVENT\",project=\"TSG-OLAP\"})", + "legend":"{{datacenter}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"DataCenter File Chunk Ingestion Rate Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":57, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "link":"", + "thresholds":[ + { + "color":"#ef2162", + "id":"7c5d0cbb" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(datacenter)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRAFFIC-FILE-STREAM-RECORD|TROUBLESHOOTING-FILE-STREAM-RECORD\"})", + "legend":"{{datacenter}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"DataCenter Metric Ingestion Rate Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":58, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "link":"", + "thresholds":[ + { + "color":"#4740ce", + "id":"23e8cec" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by(datacenter)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"DOS-SKETCH-RECORD|NETWORK-TRAFFIC-METRIC|OBJECT-STATISTICS-METRIC|POLICY-RULE-METRIC|STATISTICS-RULE-METRIC|SYS-STORAGE|TRAFFIC-TOP-METRIC|VOIP-CONVERSATION-RECORD|TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\"})", + "legend":"{{datacenter}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"DataCenter Log Ingestion IN/OUT Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":59, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "link":"", + "thresholds":[ + { + "color":"#3c11f9", + "id":"222bd78" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by (datacenter)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesInPerSec\",topic=~\"TRANSACTION-RECORD|SESSION-RECORD|BGP-RECORD|PROXY-EVENT\",project=\"TSG-OLAP\"})", + "legend":"{{datacenter}}-In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(datacenter)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesOutPerSec\",topic=~\"TRANSACTION-RECORD|SESSION-RECORD|BGP-RECORD|PROXY-EVENT\",project=\"TSG-OLAP\"})", + "legend":"{{datacenter}}-Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"DataCenter File Chunk Ingestion IN/OUT Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":60, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"desc" + }, + "link":"", + "thresholds":[ + { + "color":"#ce8c08", + "id":"243de81a" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by(datacenter)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesInPerSec\",topic=~\"TRAFFIC-FILE-STREAM-RECORD|TROUBLESHOOTING-FILE-STREAM-RECORD\",project=\"TSG-OLAP\"})", + "legend":"{{datacenter}}-In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by(datacenter)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesOutPerSec\",topic=~\"TRAFFIC-FILE-STREAM-RECORD|TROUBLESHOOTING-FILE-STREAM-RECORD\",project=\"TSG-OLAP\"})", + "legend":"{{datacenter}}-Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + } + ], + "children":[] + }, + { + "name":"[24.09] TSG OLAP Data Flow Template", + "type":"template", + "varType":0, + "param":{ + "chartShare":"none", + "variables":[ + { + "regex":"", + "expression":"", + "customOptions":"DataCenter1", + "show":true, + "name":"DataCenter", + "allOption":false, + "type":"custom", + "multi":false + } + ], + "report":{ + "schedule":{ + "repeat":1, + "etime":"", + "stime":"", + "type":2, + "nums":[] + }, + "receivers":[], + "enable":false, + "range":{ + "unit":"day", + "interval":1, + "type":"previous" + } + }, + "refresh":0, + "defaultTimeRange":4 + }, + "remark":"The template for TSG OLAP DataFlow.", + "charts":[ + { + "name":"Summary", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":0, + "param":{ + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "collapse":false + }, + "remark":"", + "x":"0.0", + "y":"1.0", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Log Ingestion Rate Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":1, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#6339a3", + "id":"068b1" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRANSACTION-RECORD|SESSION-RECORD|BGP-RECORD|PROXY-EVENT\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"Total", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRANSACTION-RECORD\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"TRANSACTION-RECORD", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"SESSION-RECORD\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"SESSION-RECORD", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"BGP-RECORD\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"BGP-RECORD", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"PROXY-EVENT\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"PROXY-EVENT", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Metric Ingestion Rate Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":2, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#b6a6ed", + "id":"f38d20e" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"DOS-SKETCH-RECORD|NETWORK-TRAFFIC-METRIC|OBJECT-STATISTICS-METRIC|POLICY-RULE-METRIC|STATISTICS-RULE-METRIC|SYS-STORAGE|TRAFFIC-TOP-METRIC|VOIP-CONVERSATION-RECORD|TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"Total", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"DOS-SKETCH-RECORD\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"DOS-SKETCH-RECORD", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"NETWORK-TRAFFIC-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"NETWORK-TRAFFIC-METRIC", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"OBJECT-STATISTICS-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"OBJECT-STATISTICS-METRIC", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"POLICY-RULE-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"POLICY-RULE-METRIC", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"STATISTICS-RULE-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"STATISTICS-RULE-METRIC", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"SYS-STORAGE\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"SYS-STORAGE", + "name":"G", + "orderNum":6, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRAFFIC-TOP-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"TRAFFIC-TOP-METRIC", + "name":"H", + "orderNum":7, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"VOIP-CONVERSATION-RECORD\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"VOIP-CONVERSATION-RECORD", + "name":"I", + "orderNum":8, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"TRAFFIC-TOP-DESTINATION-IP-METRIC", + "name":"J", + "orderNum":9, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"File Chunk Ingestion Rate Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":3, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#4ff9a7", + "id":"8e0c62c" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\".*FILE-STREAM-RECORD\",datacenter!=\"$DataCenter\"})", + "legend":"Total", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum by (topic)(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\".*FILE-STREAM-RECORD\",datacenter!=\"$DataCenter\"})", + "legend":"{{topic}}", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Traffic IN/OUT", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":4, + "param":{ + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "collapse":false + }, + "remark":"", + "x":"0.0", + "y":"5.19", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Log Ingestion IN/OUT Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":5, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#ce8c08", + "id":"6c20ee" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesInPerSec\",topic=~\"TRANSACTION-RECORD|SESSION-RECORD|BGP-RECORD|PROXY-EVENT\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesOutPerSec\",topic=~\"TRANSACTION-RECORD|SESSION-RECORD|BGP-RECORD|PROXY-EVENT\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Metric Ingestion IN/OUT Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":6, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#ce8c08", + "id":"26521b6" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesInPerSec\",topic=~\"DOS-SKETCH-RECORD|NETWORK-TRAFFIC-METRIC|OBJECT-STATISTICS-METRIC|POLICY-RULE-METRIC|STATISTICS-RULE-METRIC|SYS-STORAGE|TRAFFIC-TOP-METRIC|VOIP-CONVERSATION-RECORD|TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesOutPerSec\",topic=~\"DOS-SKETCH-RECORD|NETWORK-TRAFFIC-METRIC|OBJECT-STATISTICS-METRIC|POLICY-RULE-METRIC|STATISTICS-RULE-METRIC|SYS-STORAGE|TRAFFIC-TOP-METRIC|VOIP-CONVERSATION-RECORD|TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"File Chunk Ingestion IN/OUT Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"bytes/sec(IEC)", + "weight":7, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#ce8c08", + "id":"7e2461" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesInPerSec\",topic=~\".*FILE-STREAM-RECORD\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"In", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"BytesOutPerSec\",topic=~\".*FILE-STREAM-RECORD\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"Out", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Kafka Quotas Produce Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":8, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#7f9600", + "id":"4e8b9a" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum by (user, client_id) (kafka_server_Produce_byte_rate{datacenter!=\"$DataCenter\"})", + "legend":"", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Kafka Quotas Fetch Bytes/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":9, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#7f9600", + "id":"84b343b" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"4.0", + "elements":[ + { + "expression":"sum by (user, client_id) (kafka_server_Fetch_byte_rate{datacenter!=\"$DataCenter\"})", + "legend":"", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Logs", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":10, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":false + }, + "remark":"", + "x":"0.0", + "y":"11.38", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Session Record Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":11, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#2ce059", + "id":"3bff091" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"SESSION-RECORD\"})", + "legend":"Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"SESSION-RECORD-PROCESSED\"})", + "legend":"Transform", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate( flink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=~\"etl_session_record_kafka_to_clickhouse|session_record_processed_kafka_to_clickhouse\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":" sum(\n rate(\n flink_taskmanager_job_task_operator_internal_metrics_dropped_events{datacenter=\"$DataCenter\",job_name=~\"etl_session_record_kafka_to_clickhouse|session_record_processed_kafka_to_clickhouse\"}[5m]\n )\n )", + "legend":"Dropped", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=~\"etl_session_record_kafka_to_clickhouse |session_record_processed_kafka_to_clickhouse\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Error", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Transaction Record Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":12, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#ed0eaa", + "id":"1d4f518" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"0.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"TRANSACTION-RECORD\"})", + "legend":"Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"TRANSACTION-RECORD-PROCESSED\"})", + "legend":"Transform", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate( flink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=~\"etl_transaction_record_kafka_to_clickhouse|transaction_record_processed_kafka_to_clickhouse\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=~\"etl_transaction_record_kafka_to_clickhouse|transaction_record_processed_kafka_to_clickhouse\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Dropped", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=~\"etl_transaction_record_kafka_to_clickhouse|transaction_record_processed_kafka_to_clickhouse\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Error", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Proxy Event Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":13, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#2a16ff", + "id":"712c74" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"PROXY-EVENT\"})", + "legend":"Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"PROXY-EVENT-PROCESSED\"})", + "legend":"Transform", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate( flink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=~\"etl_proxy_event_kafka_to_clickhouse|proxy_event_processed_kafka_to_clickhouse\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=~\"etl_proxy_event_kafka_to_clickhouse|proxy_event_processed_kafka_to_clickhouse\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Dropped", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=~\"etl_proxy_event_kafka_to_clickhouse|proxy_event_processed_kafka_to_clickhouse\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Error", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"DoS Event Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":14, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#6936aa", + "id":"10312c1" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"", + "x":"0.0", + "y":"4.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"DOS-SKETCH-RECORD\"})", + "legend":"Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"DOS-EVENT\"})", + "legend":"Detection", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate( flink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=\"dos_event_kafka_to_clickhouse\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=\"dos_event_kafka_to_clickhouse\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Dropped", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=\"dos_event_kafka_to_clickhouse\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Error", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"VoIP Record Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":15, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#c02bdb", + "id":"1ba4a14" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"Collecting and processing VoIP Records per second", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"VOIP-RECORD\"}) ", + "legend":"Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"VOIP-RECORD-PROCESSED\"})", + "legend":"Transform", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=\"VOIP-CONVERSATION-RECORD\"})", + "legend":"Correlation", + "name":"C", + "orderNum":2, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate( flink_taskmanager_job_task_operator_internal_metrics_out_events{job_name=~\"correlation_sip_rtp_session|etl_voip_conversation_record_kafka_to_clickhouse\",operator_name=\"Sink:_clickhouse_sink\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Sink", + "name":"D", + "orderNum":3, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_dropped_events{job_name=~\"correlation_sip_rtp_session|etl_voip_conversation_record_kafka_to_clickhouse\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Dropped", + "name":"E", + "orderNum":4, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(\n rate(\nflink_taskmanager_job_task_operator_internal_metrics_error_events{job_name=~\"correlation_sip_rtp_session|etl_voip_conversation_record_kafka_to_clickhouse\",datacenter=\"$DataCenter\"}[5m]\n )\n)", + "legend":"Error", + "name":"F", + "orderNum":5, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + }, + { + "name":"Metrics", + "span":"12.0", + "height":"1.0", + "type":"Group", + "unit":"short", + "weight":30, + "param":{ + "thresholds":[], + "visibility":{ + "result":"show", + "varName":"", + "varValue":"", + "operator":"equal" + }, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "enable":{ + "visibility":false, + "repeat":false + }, + "showHeader":1, + "repeat":{ + "variable":"" + }, + "link":"", + "valueMapping":[], + "dataLink":[], + "collapse":true + }, + "remark":"", + "x":"0.0", + "y":"17.58", + "elements":[], + "datasource":"misc", + "children":[ + { + "name":"Druid Task Consumer Lag", + "span":"12.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":31, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[ + "max", + "avg" + ], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#04c0f9", + "id":"b03f3c4" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"zero" + }, + "remark":"The Consumer Lag in each Datasource. Consumer Lag tells us how far behind each Consumer (Group) is in each Partition.  The smaller the lag the more real-time the data consumption.", + "x":"0.0", + "y":"0.0", + "elements":[ + { + "expression":"sum by (dataSource) (irate(ingest_kafka_lag{service=\"coordinator\"}[5m]))", + "legend":"{{dataSource}}", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Network Traffic Metric Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":32, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#d058fc", + "id":"aed88d9" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"NETWORK-TRAFFIC-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"NC-Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"NETWORK-TRAFFIC-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"DT-Sink", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"DoS Sketch Record Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":33, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#d058fc", + "id":"35823cc2" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"6.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"DOS-SKETCH-RECORD\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"NC-Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"DOS-SKETCH-RECORD\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"DT-Sink", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Object Statistics Metric Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":34, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#d058fc", + "id":"61c3a26" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"2.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"OBJECT-STATISTICS-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"NC-Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"OBJECT-STATISTICS-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"DT-Sink", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Statistics Rule Metric Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":35, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#d058fc", + "id":"ab73629" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"0.0", + "y":"4.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"STATISTICS-RULE-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"NC-Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"STATISTICS-RULE-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"DT-Sink", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"TopK Metric Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":36, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#d058fc", + "id":"800e5f4" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"4.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRAFFIC-TOP-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"NC-Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRAFFIC-TOP-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"DT-Sink", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + }, + { + "name":"Traffic Top Destination IP Metric Message/sec", + "span":"6.0", + "height":"2.0", + "type":"Line chart", + "unit":"short", + "weight":37, + "param":{ + "rightYAxis":{ + "elementNames":[], + "unit":2, + "style":"line", + "label":"" + }, + "stack":0, + "color":{ + "mode":"palette", + "paletteColors":[ + "#3685FF", + "#00DCA2", + "#00BFD0", + "#954Eff", + "#FFCB01", + "#f65A96", + "#FF9094", + "#00CCF5", + "#FF8BEA", + "#4D7693", + "#72577C", + "#99D750", + "#DD8270", + "#C475EE", + "#7E83FB", + "#7EB090", + "#CF6684", + "#4E55FF", + "#FF8D00", + "#FF5200" + ] + }, + "visibility":{ + "result":"", + "varName":"", + "varValue":"", + "operator":"" + }, + "legend":{ + "values":[], + "show":true, + "placement":"bottom" + }, + "showHeader":1, + "tooltip":{ + "mode":"all", + "sort":"none" + }, + "link":"", + "thresholds":[ + { + "color":"#d058fc", + "id":"0b21f08" + } + ], + "thresholdShow":true, + "enable":{ + "rightYAxis":false, + "thresholds":false, + "visibility":false, + "legend":true, + "tooltip":true, + "valueMapping":false + }, + "valueMapping":[], + "dataLink":[], + "nullType":"null" + }, + "remark":"", + "x":"6.0", + "y":"6.0", + "elements":[ + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\",datacenter=\"$DataCenter\"})", + "legend":"NC-Ingestion", + "name":"A", + "orderNum":0, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + }, + { + "expression":"sum(kafka_server_BrokerTopicMetrics_OneMinuteRate{name=\"MessagesInPerSec\",topic=~\"TRAFFIC-TOP-DESTINATION-IP-METRIC\",project=\"TSG-OLAP\",datacenter!=\"$DataCenter\"})", + "legend":"DT-Sink", + "name":"B", + "orderNum":1, + "id":"", + "state":1, + "type":"expert", + "queryType":1 + } + ], + "datasource":"metrics" + } + ] + } + ], + "children":[] + } +] \ No newline at end of file diff --git a/monitoring-templates/TSG-OLAP-Expression.xlsx b/monitoring-templates/TSG-OLAP-Expression.xlsx new file mode 100644 index 0000000..2187b9d Binary files /dev/null and b/monitoring-templates/TSG-OLAP-Expression.xlsx differ diff --git a/monitoring-templates/TSG-OLAP-Module.xlsx b/monitoring-templates/TSG-OLAP-Module.xlsx new file mode 100644 index 0000000..18c84e0 Binary files /dev/null and b/monitoring-templates/TSG-OLAP-Module.xlsx differ